{ "analysis_source_sha256": { "scripts/stress_report.py": "581d131e4d4a878f5837cd71312a267fd0752b5349bf5c7fe1116d020d353468", "shared_evidence/analysis.py": "a8b78a46ef82c942840116136b56ce5dd7e9ed6368513fa033ff0a44439ae08a", "shared_evidence/simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "bootstrap_resamples": 10000, "confidence_level": 0.95, "experiments": { "exploratory-stress-exchangeable-m1000-v1": { "analysis_seed": 87809162144307544895889988141990942431130330468114153367967054774712364280931, "cells": { "exchangeable-n10-rho0-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12159287227776598, -0.11826141103303042 ], "confidence_level": 0.95, "mean": -0.11987101769575945, "replicates": 8, "sample_sd": 0.0025534235754438744, "standard_error": 0.0009027715627189817 }, "log_loss": { "bootstrap_interval": [ -0.37188080284075964, -0.362947726188501 ], "confidence_level": 0.95, "mean": -0.36729819116256385, "replicates": 8, "sample_sd": 0.00692596106664429, "standard_error": 0.0024486970182290954 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -2.5762806781431946e-05, 0.000320658482095941 ], "confidence_level": 0.95, "mean": 9.895294266632371e-05, "replicates": 8, "sample_sd": 0.0003064819751383528, "standard_error": 0.00010835774146588807 }, "log_loss": { "bootstrap_interval": [ -0.00013582443486939016, 0.0015453833853687864 ], "confidence_level": 0.95, "mean": 0.00048587916989245236, "replicates": 8, "sample_sd": 0.0014522388696016708, "standard_error": 0.0005134439762990139 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.5762806781431946e-05, 0.000320658482095941 ], "confidence_level": 0.95, "mean": 9.895294266632371e-05, "replicates": 8, "sample_sd": 0.0003064819751383528, "standard_error": 0.00010835774146588807 }, "log_loss": { "bootstrap_interval": [ -0.00013582443486939016, 0.0015453833853687864 ], "confidence_level": 0.95, "mean": 0.00048587916989245236, "replicates": 8, "sample_sd": 0.0014522388696016708, "standard_error": 0.0005134439762990139 } } }, "discount": { "bootstrap_interval": [ 0.9188656798795053, 0.9943389640044609 ], "confidence_level": 0.95, "mean": 0.9618781343394198, "replicates": 8, "sample_sd": 0.058390981727861765, "standard_error": 0.02064432956995542 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8279501821603419, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16226085196507728, 0.16351987509928592 ], "confidence_level": 0.95, "mean": 0.16286718405531359, "replicates": 8, "sample_sd": 0.0009869974930393046, "standard_error": 0.0003489563101711072 }, "log_loss": { "bootstrap_interval": [ 0.5127858534556431, 0.5155002735683808 ], "confidence_level": 0.95, "mean": 0.5141336851159383, "replicates": 8, "sample_sd": 0.0020831203546805145, "standard_error": 0.0007364942644111588 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04111476293553363, 0.04455493142243295 ], "confidence_level": 0.95, "mean": 0.0428972134168878, "replicates": 8, "sample_sd": 0.0026724147894993788, "standard_error": 0.0009448413098991152 }, "log_loss": { "bootstrap_interval": [ 0.14136599608447764, 0.15117044847125802 ], "confidence_level": 0.95, "mean": 0.14634961478348196, "replicates": 8, "sample_sd": 0.007678135144110263, "standard_error": 0.002714630713633558 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19338804745350063, 0.1991166389233976 ], "confidence_level": 0.95, "mean": 0.19616138579072515, "replicates": 8, "sample_sd": 0.004520943891208201, "standard_error": 0.001598395041418608 }, "log_loss": { "bootstrap_interval": [ 0.5684065550285979, 0.5816649286403046 ], "confidence_level": 0.95, "mean": 0.5749023486997822, "replicates": 8, "sample_sd": 0.010327742370136919, "standard_error": 0.0036514083321357205 } }, "learned": { "brier": { "bootstrap_interval": [ 0.041303004979373364, 0.04455309529046864 ], "confidence_level": 0.95, "mean": 0.042996166359554125, "replicates": 8, "sample_sd": 0.002514551289274491, "standard_error": 0.0008890281341436842 }, "log_loss": { "bootstrap_interval": [ 0.14229196685802775, 0.15134275249329393 ], "confidence_level": 0.95, "mean": 0.14683549395337442, "replicates": 8, "sample_sd": 0.006988559573334994, "standard_error": 0.0024708289325156697 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.041107075374758106, 0.04452084329997068 ], "confidence_level": 0.95, "mean": 0.0428972134168878, "replicates": 8, "sample_sd": 0.0026724147894993788, "standard_error": 0.0009448413098991152 }, "log_loss": { "bootstrap_interval": [ 0.14150566263400718, 0.15133612368429392 ], "confidence_level": 0.95, "mean": 0.14634961478348196, "replicates": 8, "sample_sd": 0.007678135144110263, "standard_error": 0.002714630713633558 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05687910923605548, -0.055945789448563515 ], "confidence_level": 0.95, "mean": -0.056404139543328916, "replicates": 8, "sample_sd": 0.0007214095250926541, "standard_error": 0.00025505678360279125 }, "log_loss": { "bootstrap_interval": [ -0.26018878904814763, -0.25688426796849 ], "confidence_level": 0.95, "mean": -0.2585042312547758, "replicates": 8, "sample_sd": 0.0025748958457338113, "standard_error": 0.0009103631566837241 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 5.9452502433805165e-06, 4.10950197530262e-05 ], "confidence_level": 0.95, "mean": 2.079077597372388e-05, "replicates": 8, "sample_sd": 2.894363051978768e-05, "standard_error": 1.0233118706349892e-05 }, "log_loss": { "bootstrap_interval": [ -0.00013955834684735834, 0.00021954207968646398 ], "confidence_level": 0.95, "mean": 4.21262049532776e-05, "replicates": 8, "sample_sd": 0.00028018256910276363, "standard_error": 9.90594972914163e-05 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.9452502433805165e-06, 4.10950197530262e-05 ], "confidence_level": 0.95, "mean": 2.079077597372388e-05, "replicates": 8, "sample_sd": 2.894363051978768e-05, "standard_error": 1.0233118706349892e-05 }, "log_loss": { "bootstrap_interval": [ -0.00013955834684735834, 0.00021954207968646398 ], "confidence_level": 0.95, "mean": 4.21262049532776e-05, "replicates": 8, "sample_sd": 0.00028018256910276363, "standard_error": 9.90594972914163e-05 } } }, "discount": { "bootstrap_interval": [ 0.782291377013384, 0.9837323927943001 ], "confidence_level": 0.95, "mean": 0.8969033566253012, "replicates": 8, "sample_sd": 0.16418017830266307, "standard_error": 0.058046458707114765 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.564582754026768, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05654397841592769, 0.05759500201738975 ], "confidence_level": 0.95, "mean": 0.05707500871595724, "replicates": 8, "sample_sd": 0.0008070415537778324, "standard_error": 0.0002853322776878165 }, "log_loss": { "bootstrap_interval": [ 0.2596990962680222, 0.2625534909186017 ], "confidence_level": 0.95, "mean": 0.26111972836937436, "replicates": 8, "sample_sd": 0.0022311742820433317, "standard_error": 0.0007888392324209331 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0004125700272015905, 0.0008975420759618578 ], "confidence_level": 0.95, "mean": 0.000650078396654601, "replicates": 8, "sample_sd": 0.0003746794735945041, "standard_error": 0.0001324691982750399 }, "log_loss": { "bootstrap_interval": [ 0.0015380356977904258, 0.0037956460390841675 ], "confidence_level": 0.95, "mean": 0.002573370909645289, "replicates": 8, "sample_sd": 0.001765914408016292, "standard_error": 0.0006243450264516739 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10601484175475506, 0.11214527659321273 ], "confidence_level": 0.95, "mean": 0.10914404534872604, "replicates": 8, "sample_sd": 0.004794727973542504, "standard_error": 0.0016951923320183688 }, "log_loss": { "bootstrap_interval": [ 0.3387736196154124, 0.3551021569913656 ], "confidence_level": 0.95, "mean": 0.3469855414655005, "replicates": 8, "sample_sd": 0.012538523242686587, "standard_error": 0.0044330374054844125 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0004450486040332909, 0.000924798798602702 ], "confidence_level": 0.95, "mean": 0.0006708691726283249, "replicates": 8, "sample_sd": 0.00037157998735281943, "standard_error": 0.0001313733644051951 }, "log_loss": { "bootstrap_interval": [ 0.0016771360081151135, 0.0037232530338106636 ], "confidence_level": 0.95, "mean": 0.0026154971145985664, "replicates": 8, "sample_sd": 0.001575491541671288, "standard_error": 0.0005570203764089079 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.00041833498755505284, 0.0008992388462373519 ], "confidence_level": 0.95, "mean": 0.000650078396654601, "replicates": 8, "sample_sd": 0.0003746794735945041, "standard_error": 0.0001324691982750399 }, "log_loss": { "bootstrap_interval": [ 0.001532116361709866, 0.003814253194991253 ], "confidence_level": 0.95, "mean": 0.002573370909645289, "replicates": 8, "sample_sd": 0.001765914408016292, "standard_error": 0.0006243450264516739 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.1-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07882894096926425, -0.0742850593069642 ], "confidence_level": 0.95, "mean": -0.0767381925383314, "replicates": 8, "sample_sd": 0.0035058588036752827, "standard_error": 0.0012395082669806746 }, "log_loss": { "bootstrap_interval": [ -0.23742208241748336, -0.22031945249316415 ], "confidence_level": 0.95, "mean": -0.22950315255651588, "replicates": 8, "sample_sd": 0.013255790284085573, "standard_error": 0.004686629599931829 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.005878204192889397, -0.004592174468619708 ], "confidence_level": 0.95, "mean": -0.005162266414012713, "replicates": 8, "sample_sd": 0.0009800852726736422, "standard_error": 0.00034651247122429943 }, "log_loss": { "bootstrap_interval": [ -0.06671220068523685, -0.04820588641299394 ], "confidence_level": 0.95, "mean": -0.05661494158910649, "replicates": 8, "sample_sd": 0.014384918821593607, "standard_error": 0.005085836822783419 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.87143319651533e-05, 4.1008783701459994e-05 ], "confidence_level": 0.95, "mean": 1.9920789022000174e-06, "replicates": 8, "sample_sd": 6.191175770603895e-05, "standard_error": 2.1889111854559316e-05 }, "log_loss": { "bootstrap_interval": [ -0.00046843908931676337, 0.00038783341596271975 ], "confidence_level": 0.95, "mean": -5.764936334438919e-06, "replicates": 8, "sample_sd": 0.0006600654216206676, "standard_error": 0.00023336836782736578 } } }, "discount": { "bootstrap_interval": [ 0.5063511303213655, 0.5388671302937276 ], "confidence_level": 0.95, "mean": 0.5227201712100406, "replicates": 8, "sample_sd": 0.02494180439833205, "standard_error": 0.008818259512544524 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.5635215519105774, "discount_min": 0.4831536903823027, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16366367078527366, 0.1667187379157728 ], "confidence_level": 0.95, "mean": 0.16507311136599556, "replicates": 8, "sample_sd": 0.0023758877538963834, "standard_error": 0.0008400031710591038 }, "log_loss": { "bootstrap_interval": [ 0.513454604229918, 0.5199956178522432 ], "confidence_level": 0.95, "mean": 0.5165309521303274, "replicates": 8, "sample_sd": 0.005072986351012367, "standard_error": 0.0017935715248338221 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08984612825775763, 0.09748668267647137 ], "confidence_level": 0.95, "mean": 0.09349718524167687, "replicates": 8, "sample_sd": 0.00601049273337046, "standard_error": 0.0021250300850193596 }, "log_loss": { "bootstrap_interval": [ 0.3248465138768477, 0.36401116647939397 ], "confidence_level": 0.95, "mean": 0.343642741162918, "replicates": 8, "sample_sd": 0.03072201628110873, "standard_error": 0.01086187302204775 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19137965994816492, 0.198645442308854 ], "confidence_level": 0.95, "mean": 0.1949771317143506, "replicates": 8, "sample_sd": 0.0056748999697914014, "standard_error": 0.002006380125597417 }, "log_loss": { "bootstrap_interval": [ 0.5634350472392222, 0.5807108301143029 ], "confidence_level": 0.95, "mean": 0.5721751720696512, "replicates": 8, "sample_sd": 0.013264221943656803, "standard_error": 0.004689610641761566 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08511283009347775, 0.09185725155947008 ], "confidence_level": 0.95, "mean": 0.08833491882766416, "replicates": 8, "sample_sd": 0.00523165499015184, "standard_error": 0.0018496693601824032 }, "log_loss": { "bootstrap_interval": [ 0.27650781629577836, 0.2983516750182149 ], "confidence_level": 0.95, "mean": 0.2870277995738115, "replicates": 8, "sample_sd": 0.016845507234669637, "standard_error": 0.005955786199080972 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08510768428399863, 0.09186529599664228 ], "confidence_level": 0.95, "mean": 0.08833292674876196, "replicates": 8, "sample_sd": 0.005250795862867426, "standard_error": 0.0018564366806299128 }, "log_loss": { "bootstrap_interval": [ 0.27645307876345554, 0.2983612651970156 ], "confidence_level": 0.95, "mean": 0.28703356451014594, "replicates": 8, "sample_sd": 0.017236412051548442, "standard_error": 0.006093991922487717 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5263157894736842 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5263157894736842, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.1-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05372752005575634, -0.05163153569282978 ], "confidence_level": 0.95, "mean": -0.05262269480668551, "replicates": 8, "sample_sd": 0.0016375782203639857, "standard_error": 0.0005789713321713864 }, "log_loss": { "bootstrap_interval": [ -0.2386884791344861, -0.23015486796588816 ], "confidence_level": 0.95, "mean": -0.23453121263135507, "replicates": 8, "sample_sd": 0.006800566321504092, "standard_error": 0.0024043632809221988 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0005931658156371372, -0.00029266964754180746 ], "confidence_level": 0.95, "mean": -0.00044084551007006034, "replicates": 8, "sample_sd": 0.00023063343026746993, "standard_error": 8.154123125522135e-05 }, "log_loss": { "bootstrap_interval": [ -0.01026926600251722, -0.0036391340543457966 ], "confidence_level": 0.95, "mean": -0.006693216260663605, "replicates": 8, "sample_sd": 0.005284579896530364, "standard_error": 0.0018683811402793618 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.3644722961699018e-05, 9.629197863232621e-05 ], "confidence_level": 0.95, "mean": 5.123479204621967e-05, "replicates": 8, "sample_sd": 6.322838722764974e-05, "standard_error": 2.235461068608001e-05 }, "log_loss": { "bootstrap_interval": [ -0.00018033445845348892, 0.0006125196531907456 ], "confidence_level": 0.95, "mean": 0.0001954827867756536, "replicates": 8, "sample_sd": 0.0006119711293815157, "standard_error": 0.00021636446773802987 } } }, "discount": { "bootstrap_interval": [ 0.5142999139607433, 0.5959936108345447 ], "confidence_level": 0.95, "mean": 0.5571688973913268, "replicates": 8, "sample_sd": 0.06452472845700485, "standard_error": 0.022812936523084363 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.6299623119794908, "discount_min": 0.45989594934313605, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05975959956171085, 0.062085807539623054 ], "confidence_level": 0.95, "mean": 0.06086930571036999, "replicates": 8, "sample_sd": 0.0018045346728342694, "standard_error": 0.0006379993520236799 }, "log_loss": { "bootstrap_interval": [ 0.26033770479827856, 0.2664008075922569 ], "confidence_level": 0.95, "mean": 0.26311952025199076, "replicates": 8, "sample_sd": 0.00472867680270805, "standard_error": 0.0016718397166171919 } }, "independent": { "brier": { "bootstrap_interval": [ 0.007416895132910158, 0.010038815323783857 ], "confidence_level": 0.95, "mean": 0.008687456413754546, "replicates": 8, "sample_sd": 0.002045206186697223, "standard_error": 0.0007230895817691433 }, "log_loss": { "bootstrap_interval": [ 0.02833341769035193, 0.04370874095677069 ], "confidence_level": 0.95, "mean": 0.0352815238812993, "replicates": 8, "sample_sd": 0.01204358930571195, "standard_error": 0.004258051833947351 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10505935478674364, 0.11249329588356273 ], "confidence_level": 0.95, "mean": 0.10867614770258291, "replicates": 8, "sample_sd": 0.005710394374144836, "standard_error": 0.0020189292926036623 }, "log_loss": { "bootstrap_interval": [ 0.33488611756484166, 0.3545023253821599 ], "confidence_level": 0.95, "mean": 0.34476458837454727, "replicates": 8, "sample_sd": 0.015212626970181132, "standard_error": 0.0053784758451382204 } }, "learned": { "brier": { "bootstrap_interval": [ 0.007140437481857541, 0.009568355339986156 ], "confidence_level": 0.95, "mean": 0.008246610903684486, "replicates": 8, "sample_sd": 0.001856087854689388, "standard_error": 0.0006562261542644287 }, "log_loss": { "bootstrap_interval": [ 0.024481405584247058, 0.03357782059143827 ], "confidence_level": 0.95, "mean": 0.028588307620635696, "replicates": 8, "sample_sd": 0.007001550833218211, "standard_error": 0.0024754220364954596 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.007088924159957441, 0.00943241169241412 ], "confidence_level": 0.95, "mean": 0.008195376111638267, "replicates": 8, "sample_sd": 0.001866531772636329, "standard_error": 0.0006599186368656477 }, "log_loss": { "bootstrap_interval": [ 0.024278106605356647, 0.03356239375787757 ], "confidence_level": 0.95, "mean": 0.02839282483386004, "replicates": 8, "sample_sd": 0.007223868492216353, "standard_error": 0.0025540231986230116 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5263157894736842 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5263157894736842, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.25-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.042287160975583354, -0.03819350420199874 ], "confidence_level": 0.95, "mean": -0.04034744937426969, "replicates": 8, "sample_sd": 0.0031949372541639895, "standard_error": 0.0011295808989424424 }, "log_loss": { "bootstrap_interval": [ -0.126608082687165, -0.11190416608128355 ], "confidence_level": 0.95, "mean": -0.11960513270257894, "replicates": 8, "sample_sd": 0.011499475360985978, "standard_error": 0.0040656785039204025 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.02127199089364283, -0.01860330238860523 ], "confidence_level": 0.95, "mean": -0.019859402073391194, "replicates": 8, "sample_sd": 0.002082888012558166, "standard_error": 0.0007364121190660249 }, "log_loss": { "bootstrap_interval": [ -0.26708984353083365, -0.22962604229163208 ], "confidence_level": 0.95, "mean": -0.2473493585584398, "replicates": 8, "sample_sd": 0.02924780426412311, "standard_error": 0.010340660364989136 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -4.891879502490943e-05, 0.0001230962336966064 ], "confidence_level": 0.95, "mean": 3.1316778301902076e-05, "replicates": 8, "sample_sd": 0.00013581297268934584, "standard_error": 4.801713698086991e-05 }, "log_loss": { "bootstrap_interval": [ -0.0003139379249150892, 0.0002863339144955283 ], "confidence_level": 0.95, "mean": -4.8016858313675026e-05, "replicates": 8, "sample_sd": 0.00046585218575226513, "standard_error": 0.0001647036197880009 } } }, "discount": { "bootstrap_interval": [ 0.2969361000387628, 0.32033984059542886 ], "confidence_level": 0.95, "mean": 0.30911978079337055, "replicates": 8, "sample_sd": 0.017886028479547247, "standard_error": 0.006323666013191786 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3313985134012234, "discount_min": 0.27713010578315633, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16755229034987484, 0.1720473037786739 ], "confidence_level": 0.95, "mean": 0.16965703430682746, "replicates": 8, "sample_sd": 0.003445103379390279, "standard_error": 0.0012180279807277785 }, "log_loss": { "bootstrap_interval": [ 0.5189173816565228, 0.5285527811459991 ], "confidence_level": 0.95, "mean": 0.523252900884706, "replicates": 8, "sample_sd": 0.0074747776685145095, "standard_error": 0.00264273298863419 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14508082236828054, 0.1537461210696869 ], "confidence_level": 0.95, "mean": 0.14916898700594897, "replicates": 8, "sample_sd": 0.0067454024215399845, "standard_error": 0.0023848598970515407 }, "log_loss": { "bootstrap_interval": [ 0.6224392497512293, 0.6835035472686601 ], "confidence_level": 0.95, "mean": 0.6509971267405669, "replicates": 8, "sample_sd": 0.04679349061795918, "standard_error": 0.016543997265674012 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19036380040311196, 0.19873280752763267 ], "confidence_level": 0.95, "mean": 0.1943748998290891, "replicates": 8, "sample_sd": 0.006438871471085747, "standard_error": 0.002276484840196666 }, "log_loss": { "bootstrap_interval": [ 0.5616918160473843, 0.5807420743640554 ], "confidence_level": 0.95, "mean": 0.5710071403004052, "replicates": 8, "sample_sd": 0.014887286419211677, "standard_error": 0.0052634505902454855 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12542370772378467, 0.1335953260472863 ], "confidence_level": 0.95, "mean": 0.1293095849325578, "replicates": 8, "sample_sd": 0.006292160122883192, "standard_error": 0.0022246145456011426 }, "log_loss": { "bootstrap_interval": [ 0.39257387999807153, 0.4161390320376776 ], "confidence_level": 0.95, "mean": 0.40364776818212705, "replicates": 8, "sample_sd": 0.0181781946860092, "standard_error": 0.006426962366103184 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.12544023574853913, 0.13339792099737824 ], "confidence_level": 0.95, "mean": 0.12927826815425586, "replicates": 8, "sample_sd": 0.006232442765817866, "standard_error": 0.0022035012715334273 }, "log_loss": { "bootstrap_interval": [ 0.3926262917362475, 0.4160107078666965 ], "confidence_level": 0.95, "mean": 0.4036957850404407, "replicates": 8, "sample_sd": 0.01819782008051804, "standard_error": 0.0064339009908735145 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3076923076923077 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3076923076923077, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.25-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.039998651452643696, -0.03734934136086183 ], "confidence_level": 0.95, "mean": -0.03870499831859068, "replicates": 8, "sample_sd": 0.002073172865096012, "standard_error": 0.0007329772957406668 }, "log_loss": { "bootstrap_interval": [ -0.1745500736128062, -0.1653520395690965 ], "confidence_level": 0.95, "mean": -0.1707613765909098, "replicates": 8, "sample_sd": 0.007388134868696788, "standard_error": 0.002612100132988141 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.005389121273579874, -0.002846307049747702 ], "confidence_level": 0.95, "mean": -0.004164717378914992, "replicates": 8, "sample_sd": 0.00199850908681507, "standard_error": 0.0007065796637749353 }, "log_loss": { "bootstrap_interval": [ -0.10627446494730848, -0.0739014921321409 ], "confidence_level": 0.95, "mean": -0.08844906289982431, "replicates": 8, "sample_sd": 0.025284572167793704, "standard_error": 0.008939446219623786 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -8.367017793775187e-05, 7.017952402208882e-05 ], "confidence_level": 0.95, "mean": 2.4883142111351182e-06, "replicates": 8, "sample_sd": 0.00011873556063557863, "standard_error": 4.197936004670207e-05 }, "log_loss": { "bootstrap_interval": [ -0.0003849626297365306, 0.0006838226941904349 ], "confidence_level": 0.95, "mean": 0.00017419823789945575, "replicates": 8, "sample_sd": 0.0008275039001921066, "standard_error": 0.00029256680964207725 } } }, "discount": { "bootstrap_interval": [ 0.2847458892988133, 0.3126738219987901 ], "confidence_level": 0.95, "mean": 0.29855970396732145, "replicates": 8, "sample_sd": 0.021809816612414015, "standard_error": 0.007710934611536482 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3371471658613697, "discount_min": 0.2673898682863185, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06584427182634647, 0.06922093758776768 ], "confidence_level": 0.95, "mean": 0.06746128729183042, "replicates": 8, "sample_sd": 0.0026254189398763543, "standard_error": 0.0009282257679210834 }, "log_loss": { "bootstrap_interval": [ 0.26519202016550464, 0.27398894304210003 ], "confidence_level": 0.95, "mean": 0.2693228194864611, "replicates": 8, "sample_sd": 0.006929454732237132, "standard_error": 0.0024499322155450438 } }, "independent": { "brier": { "bootstrap_interval": [ 0.02932496101583057, 0.03650868077962425 ], "confidence_level": 0.95, "mean": 0.03292100635215474, "replicates": 8, "sample_sd": 0.0056763048222710005, "standard_error": 0.002006876815954862 }, "log_loss": { "bootstrap_interval": [ 0.16570465787672153, 0.21210912795412176 ], "confidence_level": 0.95, "mean": 0.18701050579537556, "replicates": 8, "sample_sd": 0.036603262377178175, "standard_error": 0.012941207520226556 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10451205191640398, 0.112328089813116 ], "confidence_level": 0.95, "mean": 0.10829839718771324, "replicates": 8, "sample_sd": 0.006040941932025175, "standard_error": 0.0021357955024445823 }, "log_loss": { "bootstrap_interval": [ 0.3330582268741134, 0.3545241414031359 ], "confidence_level": 0.95, "mean": 0.3440690215177385, "replicates": 8, "sample_sd": 0.016362456544759204, "standard_error": 0.005785001989834719 } }, "learned": { "brier": { "bootstrap_interval": [ 0.02635185474230176, 0.0313882618960302 ], "confidence_level": 0.95, "mean": 0.028756288973239744, "replicates": 8, "sample_sd": 0.003911851919605812, "standard_error": 0.0013830485096754412 }, "log_loss": { "bootstrap_interval": [ 0.09168295639546298, 0.10664474664558178 ], "confidence_level": 0.95, "mean": 0.09856144289555126, "replicates": 8, "sample_sd": 0.01159605067586754, "standard_error": 0.004099823033944392 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.026289893495516824, 0.03137824574914017 ], "confidence_level": 0.95, "mean": 0.02875380065902861, "replicates": 8, "sample_sd": 0.004012081327048222, "standard_error": 0.00141848495651386 }, "log_loss": { "bootstrap_interval": [ 0.09108657395715287, 0.10699647930590622 ], "confidence_level": 0.95, "mean": 0.0983872446576518, "replicates": 8, "sample_sd": 0.012257992417614345, "standard_error": 0.004333854781114192 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3076923076923077 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3076923076923077, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.5-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.014811407369117756, -0.011628212431294353 ], "confidence_level": 0.95, "mean": -0.013269831160507693, "replicates": 8, "sample_sd": 0.002526615930233302, "standard_error": 0.0008932936288609623 }, "log_loss": { "bootstrap_interval": [ -0.04463356230594175, -0.03469685723254521 ], "confidence_level": 0.95, "mean": -0.03998798518402992, "replicates": 8, "sample_sd": 0.007803164978181533, "standard_error": 0.00275883543539477 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04706719112732125, -0.039967117238909576 ], "confidence_level": 0.95, "mean": -0.04333035373273128, "replicates": 8, "sample_sd": 0.005512255645027921, "standard_error": 0.0019488766731165345 }, "log_loss": { "bootstrap_interval": [ -0.6750051540854269, -0.6010943545643407 ], "confidence_level": 0.95, "mean": -0.6351183104166535, "replicates": 8, "sample_sd": 0.05786736685487116, "standard_error": 0.020459203756244528 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00014094720799660054, 0.00011572386159299293 ], "confidence_level": 0.95, "mean": -1.3153495853901465e-05, "replicates": 8, "sample_sd": 0.00019819776891020436, "standard_error": 7.007349320622489e-05 }, "log_loss": { "bootstrap_interval": [ -0.00046180932569125997, 0.00041089064791571817 ], "confidence_level": 0.95, "mean": -3.113487465908338e-05, "replicates": 8, "sample_sd": 0.0006776744639633522, "standard_error": 0.00023959410445272247 } } }, "discount": { "bootstrap_interval": [ 0.17445101988656375, 0.19042294889137104 ], "confidence_level": 0.95, "mean": 0.18277153238164282, "replicates": 8, "sample_sd": 0.012360255046014717, "standard_error": 0.0043700100801161235 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.1984184738892028, "discount_min": 0.16016384581568766, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17503082368140221, 0.18112409522976688 ], "confidence_level": 0.95, "mean": 0.17779953231487772, "replicates": 8, "sample_sd": 0.0047259245567844305, "standard_error": 0.0016708666507391499 }, "log_loss": { "bootstrap_interval": [ 0.5306811415696904, 0.5444330876974448 ], "confidence_level": 0.95, "mean": 0.5368396273753473, "replicates": 8, "sample_sd": 0.010495103869377116, "standard_error": 0.003710579557646866 } }, "independent": { "brier": { "bootstrap_interval": [ 0.20064560344625185, 0.21599092848312937 ], "confidence_level": 0.95, "mean": 0.2078600548871013, "replicates": 8, "sample_sd": 0.012182293569576567, "standard_error": 0.004307091196726431 }, "log_loss": { "bootstrap_interval": [ 1.0871573424230787, 1.1835756913154114 ], "confidence_level": 0.95, "mean": 1.1319699526079707, "replicates": 8, "sample_sd": 0.07530513151969451, "standard_error": 0.026624384577860403 } }, "individual": { "brier": { "bootstrap_interval": [ 0.18947656691361722, 0.1987614695571142 ], "confidence_level": 0.95, "mean": 0.19388169492360882, "replicates": 8, "sample_sd": 0.0071470337450813765, "standard_error": 0.002526858013258064 }, "log_loss": { "bootstrap_interval": [ 0.5597529177585946, 0.5807615563762324 ], "confidence_level": 0.95, "mean": 0.5701575733519997, "replicates": 8, "sample_sd": 0.016369939259517546, "standard_error": 0.0057876475290083735 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16028833908124082, 0.169307302459784 ], "confidence_level": 0.95, "mean": 0.16452970115437002, "replicates": 8, "sample_sd": 0.0070639176229877285, "standard_error": 0.00249747202647889 }, "log_loss": { "bootstrap_interval": [ 0.4859201256744722, 0.5092093240308385 ], "confidence_level": 0.95, "mean": 0.49685164219131733, "replicates": 8, "sample_sd": 0.017939101815338927, "standard_error": 0.00634243027101103 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.16033276170788405, 0.16933675395043998 ], "confidence_level": 0.95, "mean": 0.16454285465022392, "replicates": 8, "sample_sd": 0.006963077511913383, "standard_error": 0.002461819663300753 }, "log_loss": { "bootstrap_interval": [ 0.48622780095490414, 0.509074812333885 ], "confidence_level": 0.95, "mean": 0.49688277706597644, "replicates": 8, "sample_sd": 0.017701695011336324, "standard_error": 0.006258494290505997 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.18181818181818182 ], "oracle_method": "oracle", "population_optimal_scalar": 0.18181818181818182, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.5-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.018128140346572702, -0.01619625287980575 ], "confidence_level": 0.95, "mean": -0.017116281792554935, "replicates": 8, "sample_sd": 0.0015187395495491467, "standard_error": 0.0005369555171712021 }, "log_loss": { "bootstrap_interval": [ -0.08083417196652788, -0.07264074475834795 ], "confidence_level": 0.95, "mean": -0.0770127351329924, "replicates": 8, "sample_sd": 0.006306501281173982, "standard_error": 0.002229684910739886 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.014598409089027082, -0.012141738045643459 ], "confidence_level": 0.95, "mean": -0.013490907158769227, "replicates": 8, "sample_sd": 0.0019205858243595652, "standard_error": 0.000679029630127702 }, "log_loss": { "bootstrap_interval": [ -0.4216263229413929, -0.35092710163513224 ], "confidence_level": 0.95, "mean": -0.38464299831173654, "replicates": 8, "sample_sd": 0.0550519097259504, "standard_error": 0.019463789342244586 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.73697062137505e-06, 0.00016588362078061034 ], "confidence_level": 0.95, "mean": 8.178450017430963e-05, "replicates": 8, "sample_sd": 0.00012330089233239357, "standard_error": 4.359344854729393e-05 }, "log_loss": { "bootstrap_interval": [ -2.57035917876694e-05, 0.001145144452233319 ], "confidence_level": 0.95, "mean": 0.0005177581999361129, "replicates": 8, "sample_sd": 0.0009048544742603166, "standard_error": 0.00031991436736822907 } } }, "discount": { "bootstrap_interval": [ 0.17085116369348707, 0.18890140053244653 ], "confidence_level": 0.95, "mean": 0.17948307878329184, "replicates": 8, "sample_sd": 0.013876086630387178, "standard_error": 0.004905937476339381 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.204792404085138, "discount_min": 0.15931141083944828, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07725269434992256, 0.0821170059758382 ], "confidence_level": 0.95, "mean": 0.07950026922522321, "replicates": 8, "sample_sd": 0.003753565713529184, "standard_error": 0.0013270858848329038 }, "log_loss": { "bootstrap_interval": [ 0.2785715476880858, 0.2914896582291797 ], "confidence_level": 0.95, "mean": 0.2844527038288304, "replicates": 8, "sample_sd": 0.009974273893983795, "standard_error": 0.0035264383539239464 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07266436081886693, 0.0792865190926536 ], "confidence_level": 0.95, "mean": 0.07587489459143751, "replicates": 8, "sample_sd": 0.0050807516175395955, "standard_error": 0.001796316961143384 }, "log_loss": { "bootstrap_interval": [ 0.5499682410782406, 0.6385899275648694 ], "confidence_level": 0.95, "mean": 0.5920829670075747, "replicates": 8, "sample_sd": 0.068865289359473, "standard_error": 0.024347556547228578 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10361459276890894, 0.1119759755938168 ], "confidence_level": 0.95, "mean": 0.10774614617706736, "replicates": 8, "sample_sd": 0.006332313117656662, "standard_error": 0.002238810773045777 }, "log_loss": { "bootstrap_interval": [ 0.3320327963375327, 0.3549789219246329 ], "confidence_level": 0.95, "mean": 0.3436452340089625, "replicates": 8, "sample_sd": 0.01747682052815941, "standard_error": 0.0061789891545208885 } }, "learned": { "brier": { "bootstrap_interval": [ 0.05962683729354224, 0.06554563594575145 ], "confidence_level": 0.95, "mean": 0.06238398743266827, "replicates": 8, "sample_sd": 0.004603550602699573, "standard_error": 0.0016276009243521429 }, "log_loss": { "bootstrap_interval": [ 0.19870068215970296, 0.21739489896418396 ], "confidence_level": 0.95, "mean": 0.20743996869583806, "replicates": 8, "sample_sd": 0.014611111217406852, "standard_error": 0.0051658079112496084 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.059616600755757095, 0.0654612798547504 ], "confidence_level": 0.95, "mean": 0.062302202932493965, "replicates": 8, "sample_sd": 0.004588815630568679, "standard_error": 0.001622391324994968 }, "log_loss": { "bootstrap_interval": [ 0.19802567651919759, 0.21676747012398895 ], "confidence_level": 0.95, "mean": 0.20692221049590193, "replicates": 8, "sample_sd": 0.014730911649440175, "standard_error": 0.0052081637601895285 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.18181818181818182 ], "oracle_method": "oracle", "population_optimal_scalar": 0.18181818181818182, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.75-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0038180222432156775, -0.002045207696749308 ], "confidence_level": 0.95, "mean": -0.002955492363008738, "replicates": 8, "sample_sd": 0.001375012379218469, "standard_error": 0.000486140288780414 }, "log_loss": { "bootstrap_interval": [ -0.011664081915221947, -0.006471567938067954 ], "confidence_level": 0.95, "mean": -0.009140184919434122, "replicates": 8, "sample_sd": 0.004041738785169537, "standard_error": 0.001428970451389029 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06585623003299475, -0.055062231241338355 ], "confidence_level": 0.95, "mean": -0.060322302275378645, "replicates": 8, "sample_sd": 0.008281762728007052, "standard_error": 0.0029280452925758934 }, "log_loss": { "bootstrap_interval": [ -1.0771858582827942, -0.9669170180970815 ], "confidence_level": 0.95, "mean": -1.0182306844267528, "replicates": 8, "sample_sd": 0.08550687692754215, "standard_error": 0.030231246256774296 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00017821608307758081, 0.00015335482500657138 ], "confidence_level": 0.95, "mean": -7.479490597189081e-06, "replicates": 8, "sample_sd": 0.0002550572707186372, "standard_error": 9.01763628580407e-05 }, "log_loss": { "bootstrap_interval": [ -0.0005142036421779923, 0.0006083841615550841 ], "confidence_level": 0.95, "mean": 7.045377299616518e-05, "replicates": 8, "sample_sd": 0.0008683198642655367, "standard_error": 0.00030699743213057177 } } }, "discount": { "bootstrap_interval": [ 0.12270105537745504, 0.13516549071246248 ], "confidence_level": 0.95, "mean": 0.12914355755457088, "replicates": 8, "sample_sd": 0.009733611044335944, "standard_error": 0.003441351187441109 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.1415851113106411, "discount_min": 0.11175636727355051, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18291548896812673, 0.19028961846583256 ], "confidence_level": 0.95, "mean": 0.18628111455829632, "replicates": 8, "sample_sd": 0.00575994437723193, "standard_error": 0.0020364478641990112 }, "log_loss": { "bootstrap_interval": [ 0.5454214418141032, 0.5620183036053733 ], "confidence_level": 0.95, "mean": 0.5530362724620446, "replicates": 8, "sample_sd": 0.013064545162789182, "standard_error": 0.0046190142388630685 } }, "independent": { "brier": { "bootstrap_interval": [ 0.23449947486996164, 0.25387001425418515 ], "confidence_level": 0.95, "mean": 0.24364792447066622, "replicates": 8, "sample_sd": 0.014829815135020007, "standard_error": 0.005243131422857771 }, "log_loss": { "bootstrap_interval": [ 1.5022195839250585, 1.633625693654383 ], "confidence_level": 0.95, "mean": 1.5621267719693632, "replicates": 8, "sample_sd": 0.1018809667073219, "standard_error": 0.036020361216294094 } }, "individual": { "brier": { "bootstrap_interval": [ 0.18940192223233207, 0.1987940732519089 ], "confidence_level": 0.95, "mean": 0.19386376085344048, "replicates": 8, "sample_sd": 0.007383401697858005, "standard_error": 0.002610426704389832 }, "log_loss": { "bootstrap_interval": [ 0.5595258421927823, 0.5814067695331656 ], "confidence_level": 0.95, "mean": 0.5700788344611047, "replicates": 8, "sample_sd": 0.016928713781026974, "standard_error": 0.005985204155665166 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17917103014151226, 0.18831718950994658 ], "confidence_level": 0.95, "mean": 0.18332562219528759, "replicates": 8, "sample_sd": 0.007024933590887307, "standard_error": 0.002483689089750789 }, "log_loss": { "bootstrap_interval": [ 0.5338623282660951, 0.5559060619322647 ], "confidence_level": 0.95, "mean": 0.5438960875426104, "replicates": 8, "sample_sd": 0.016883000142263193, "standard_error": 0.0059690419436838746 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.17928657028034556, 0.18806711192913206 ], "confidence_level": 0.95, "mean": 0.18333310168588476, "replicates": 8, "sample_sd": 0.006911177938876993, "standard_error": 0.002443470393283394 }, "log_loss": { "bootstrap_interval": [ 0.5343006046079624, 0.5551307832599532 ], "confidence_level": 0.95, "mean": 0.5438256337696142, "replicates": 8, "sample_sd": 0.016563081970806304, "standard_error": 0.005855933789452891 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12903225806451613 ], "oracle_method": "oracle", "population_optimal_scalar": 0.12903225806451613, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho0.75-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0050646349740648515, -0.0037347263043651657 ], "confidence_level": 0.95, "mean": -0.004361744732568026, "replicates": 8, "sample_sd": 0.0010413909183571094, "standard_error": 0.0003681872901181992 }, "log_loss": { "bootstrap_interval": [ -0.023050812277178555, -0.01776337002247138 ], "confidence_level": 0.95, "mean": -0.0204890626319248, "replicates": 8, "sample_sd": 0.004128147506670565, "standard_error": 0.0014595205478525476 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.026347568188690863, -0.020191935768840506 ], "confidence_level": 0.95, "mean": -0.02303244371614553, "replicates": 8, "sample_sd": 0.004804393718318387, "standard_error": 0.0016986096888564914 }, "log_loss": { "bootstrap_interval": [ -0.8516368626135002, -0.7539929439415952 ], "confidence_level": 0.95, "mean": -0.7999564342658637, "replicates": 8, "sample_sd": 0.07551864015938296, "standard_error": 0.026699871281343213 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.3044262022508926e-05, 0.00021375279324509916 ], "confidence_level": 0.95, "mean": 7.652042609864948e-05, "replicates": 8, "sample_sd": 0.00019396634162112164, "standard_error": 6.85774577411208e-05 }, "log_loss": { "bootstrap_interval": [ -0.00010161890873781193, 0.0011409793337898437 ], "confidence_level": 0.95, "mean": 0.000495817653879807, "replicates": 8, "sample_sd": 0.0009597947187510522, "standard_error": 0.00033933867708795207 } } }, "discount": { "bootstrap_interval": [ 0.12214572685334125, 0.1347952212277981 ], "confidence_level": 0.95, "mean": 0.12840760490736136, "replicates": 8, "sample_sd": 0.009724790350648506, "standard_error": 0.0034382326012805304 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.14448911288567384, "discount_min": 0.1136397884762865, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09013799018120772, 0.09656080704681744 ], "confidence_level": 0.95, "mean": 0.09299322225627533, "replicates": 8, "sample_sd": 0.004933186697696637, "standard_error": 0.001744144883400281 }, "log_loss": { "bootstrap_interval": [ 0.30004955320162047, 0.31743275602246124 ], "confidence_level": 0.95, "mean": 0.30783691282578207, "replicates": 8, "sample_sd": 0.01317706672614183, "standard_error": 0.0046587966191012535 } }, "independent": { "brier": { "bootstrap_interval": [ 0.10615495991570915, 0.11842541403106756 ], "confidence_level": 0.95, "mean": 0.11166392123985283, "replicates": 8, "sample_sd": 0.009585946412221285, "standard_error": 0.003389143856086263 }, "log_loss": { "bootstrap_interval": [ 1.0312148134114418, 1.1484903732990808 ], "confidence_level": 0.95, "mean": 1.087304284459721, "replicates": 8, "sample_sd": 0.0912027621634541, "standard_error": 0.03224504579436114 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10344939913241942, 0.11179812742795978 ], "confidence_level": 0.95, "mean": 0.10749807007090667, "replicates": 8, "sample_sd": 0.006495836244050999, "standard_error": 0.0022966249288229068 }, "log_loss": { "bootstrap_interval": [ 0.3319023949890066, 0.35522701950719365 ], "confidence_level": 0.95, "mean": 0.3436335678809739, "replicates": 8, "sample_sd": 0.01819648125975606, "standard_error": 0.00643342764625372 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08527904822464002, 0.09257546841779003 ], "confidence_level": 0.95, "mean": 0.0886314775237073, "replicates": 8, "sample_sd": 0.0056445341449153676, "standard_error": 0.0019956441852543336 }, "log_loss": { "bootstrap_interval": [ 0.27748230354554265, 0.29864417596851023 ], "confidence_level": 0.95, "mean": 0.2873478501938573, "replicates": 8, "sample_sd": 0.01642363178779942, "standard_error": 0.005806630704431956 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08525823493054498, 0.09238479269004816 ], "confidence_level": 0.95, "mean": 0.08855495709760866, "replicates": 8, "sample_sd": 0.0056033940341662205, "standard_error": 0.0019810989596095894 }, "log_loss": { "bootstrap_interval": [ 0.2770520332900772, 0.2984517290143893 ], "confidence_level": 0.95, "mean": 0.2868520325399775, "replicates": 8, "sample_sd": 0.016346446824321053, "standard_error": 0.00577934169889136 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12903225806451613 ], "oracle_method": "oracle", "population_optimal_scalar": 0.12903225806451613, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho1-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.00016476224821131147, 0.00026717867559839236 ], "confidence_level": 0.95, "mean": 5.417565099974375e-05, "replicates": 8, "sample_sd": 0.0003374385605486479, "standard_error": 0.00011930254719888816 }, "log_loss": { "bootstrap_interval": [ -0.00042397014773659533, 0.0010207963816166593 ], "confidence_level": 0.95, "mean": 0.00031282080976346993, "replicates": 8, "sample_sd": 0.0011149038923301022, "standard_error": 0.0003941780513189458 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.08026316978642067, -0.06740497080527823 ], "confidence_level": 0.95, "mean": -0.07357124801034119, "replicates": 8, "sample_sd": 0.01015197337721372, "standard_error": 0.0035892646087265587 }, "log_loss": { "bootstrap_interval": [ -1.4532499785072988, -1.3156980639291311 ], "confidence_level": 0.95, "mean": -1.3781296529556823, "replicates": 8, "sample_sd": 0.10744238859999586, "standard_error": 0.03798662078296864 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00016476224821131147, 0.0002671786755983993 ], "confidence_level": 0.95, "mean": 5.417565099974722e-05, "replicates": 8, "sample_sd": 0.00033743856054865254, "standard_error": 0.0001193025471988898 }, "log_loss": { "bootstrap_interval": [ -0.00042397014773659636, 0.0010207963816166454 ], "confidence_level": 0.95, "mean": 0.00031282080976345605, "replicates": 8, "sample_sd": 0.0011149038923301054, "standard_error": 0.00039417805131894694 } } }, "discount": { "bootstrap_interval": [ 0.09369896184246476, 0.10439731549462634 ], "confidence_level": 0.95, "mean": 0.09915275236687204, "replicates": 8, "sample_sd": 0.008307976256960075, "standard_error": 0.0029373131746166497 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.10906664303559693, "discount_min": 0.08546989974102406, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1912793032000528, 0.19965439081019445 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.563058386273107, 0.5823349510593254 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348748, "standard_error": 0.005382364421255804 } }, "independent": { "brier": { "bootstrap_interval": [ 0.25892677917494633, 0.2801976022997225 ], "confidence_level": 0.95, "mean": 0.26859644088960777, "replicates": 8, "sample_sd": 0.01643327374669877, "standard_error": 0.005810039651692782 }, "log_loss": { "bootstrap_interval": [ 1.880397649114034, 2.0356823390282868 ], "confidence_level": 0.95, "mean": 1.9503377961992185, "replicates": 8, "sample_sd": 0.12245788290970924, "standard_error": 0.043295399707601816 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19117585283435953, 0.19966620330083476 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5633480204254463, 0.5823476539986959 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1911067361921412, 0.1997645099995041 ], "confidence_level": 0.95, "mean": 0.19502519287926653, "replicates": 8, "sample_sd": 0.006700670866674529, "standard_error": 0.00236904490416235 }, "log_loss": { "bootstrap_interval": [ 0.5631294456283957, 0.5831371375245222 ], "confidence_level": 0.95, "mean": 0.5722081432435364, "replicates": 8, "sample_sd": 0.0154746166590676, "standard_error": 0.005471103187944508 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19106708058246202, 0.19947980221600675 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5629655380835424, 0.5829051096112987 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1 ], "oracle_method": "oracle", "population_optimal_scalar": 0.1, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n10-rho1-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -8.62248613173796e-05, 0.00018041199733248754 ], "confidence_level": 0.95, "mean": 3.281577390299925e-05, "replicates": 8, "sample_sd": 0.00020814777778857522, "standard_error": 7.359135258160608e-05 }, "log_loss": { "bootstrap_interval": [ -0.00014587132870876875, 0.0011491910746792547 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.034757612188192, -0.028788460883125956 ], "confidence_level": 0.95, "mean": -0.03192657220851848, "replicates": 8, "sample_sd": 0.00470779309919625, "standard_error": 0.0016644562124324505 }, "log_loss": { "bootstrap_interval": [ -1.329493340131592, -1.1925413373619496 ], "confidence_level": 0.95, "mean": -1.2583237968459153, "replicates": 8, "sample_sd": 0.10540777899254526, "standard_error": 0.03726727765772083 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -8.62248613173796e-05, 0.00018041199733248754 ], "confidence_level": 0.95, "mean": 3.281577390299925e-05, "replicates": 8, "sample_sd": 0.00020814777778857522, "standard_error": 7.359135258160608e-05 }, "log_loss": { "bootstrap_interval": [ -0.00014587132870876875, 0.0011491910746792547 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } } }, "discount": { "bootstrap_interval": [ 0.0945635547250668, 0.10361530696005998 ], "confidence_level": 0.95, "mean": 0.09918768238715343, "replicates": 8, "sample_sd": 0.006989794178989661, "standard_error": 0.002471265431530923 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.10794162030120072, "discount_min": 0.08789503090367132, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.10474519763354159, 0.11286343882836762 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3349936518660344, 0.3565186609491149 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "independent": { "brier": { "bootstrap_interval": [ 0.13442456187679996, 0.14713842624653567 ], "confidence_level": 0.95, "mean": 0.1403278663450321, "replicates": 8, "sample_sd": 0.009834641894737229, "standard_error": 0.003477070987155005 }, "log_loss": { "bootstrap_interval": [ 1.5310378643225095, 1.6834426388662116 ], "confidence_level": 0.95, "mean": 1.6036334946385908, "replicates": 8, "sample_sd": 0.12142455621911223, "standard_error": 0.04293006355255071 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10482552551396881, 0.11275911246009505 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3348981931413736, 0.35651032268790356 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10468881692231152, 0.11295855079957495 ], "confidence_level": 0.95, "mean": 0.10840129413651362, "replicates": 8, "sample_sd": 0.006334716340172473, "standard_error": 0.002239660440514592 }, "log_loss": { "bootstrap_interval": [ 0.33534137222929716, 0.35697818366495065 ], "confidence_level": 0.95, "mean": 0.3453096977926756, "replicates": 8, "sample_sd": 0.017009453256656443, "standard_error": 0.006013749871028687 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10477923259163588, 0.11287103342737595 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.33500466893220954, 0.35671257437811293 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1 ], "oracle_method": "oracle", "population_optimal_scalar": 0.1, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.01904518858161095, -0.01575719160875149 ], "confidence_level": 0.95, "mean": -0.01738437102676209, "replicates": 8, "sample_sd": 0.002531612982268828, "standard_error": 0.0008950603535510935 }, "log_loss": { "bootstrap_interval": [ -0.0565723929423114, -0.04571865370197694 ], "confidence_level": 0.95, "mean": -0.0510877541420317, "replicates": 8, "sample_sd": 0.008402274719131212, "standard_error": 0.002970652715644987 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -1.0609513669310178e-05, 9.240207454366475e-05 ], "confidence_level": 0.95, "mean": 2.7264186958118192e-05, "replicates": 8, "sample_sd": 8.909821102922686e-05, "standard_error": 3.150097460517817e-05 }, "log_loss": { "bootstrap_interval": [ -5.2647013574720625e-05, 0.0003411594281977834 ], "confidence_level": 0.95, "mean": 9.617080487435425e-05, "replicates": 8, "sample_sd": 0.00033238957722489217, "standard_error": 0.00011751746202572542 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -1.0609513669310178e-05, 9.240207454366475e-05 ], "confidence_level": 0.95, "mean": 2.7264186958118192e-05, "replicates": 8, "sample_sd": 8.909821102922686e-05, "standard_error": 3.150097460517817e-05 }, "log_loss": { "bootstrap_interval": [ -5.2647013574720625e-05, 0.0003411594281977834 ], "confidence_level": 0.95, "mean": 9.617080487435425e-05, "replicates": 8, "sample_sd": 0.00033238957722489217, "standard_error": 0.00011751746202572542 } } }, "discount": { "bootstrap_interval": [ 0.9871336408942335, 1.0 ], "confidence_level": 0.95, "mean": 0.9947309668399678, "replicates": 8, "sample_sd": 0.010952986312071195, "standard_error": 0.003872465447754488 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.9696106962170628, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1735806011651042, 0.17947865520378592 ], "confidence_level": 0.95, "mean": 0.1764273116819474, "replicates": 8, "sample_sd": 0.004642656969215075, "standard_error": 0.0016414271128274817 }, "log_loss": { "bootstrap_interval": [ 0.5280121590846192, 0.5410780045630331 ], "confidence_level": 0.95, "mean": 0.5344900183059482, "replicates": 8, "sample_sd": 0.010289641850016934, "standard_error": 0.0036379377640639327 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15450087108755883, 0.16359298717230505 ], "confidence_level": 0.95, "mean": 0.15901567646822717, "replicates": 8, "sample_sd": 0.007068486600438236, "standard_error": 0.0024990874039480613 }, "log_loss": { "bootstrap_interval": [ 0.47121551135077, 0.4956252508208026 ], "confidence_level": 0.95, "mean": 0.4833060933590422, "replicates": 8, "sample_sd": 0.018654237181519016, "standard_error": 0.006595268804457162 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19324496385520162, 0.199075729998605 ], "confidence_level": 0.95, "mean": 0.19616138579072515, "replicates": 8, "sample_sd": 0.004520943891208201, "standard_error": 0.001598395041418608 }, "log_loss": { "bootstrap_interval": [ 0.5683876644441057, 0.5815370300642034 ], "confidence_level": 0.95, "mean": 0.5749023486997822, "replicates": 8, "sample_sd": 0.010327742370136919, "standard_error": 0.0036514083321357205 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15454238717968982, 0.1635134046755298 ], "confidence_level": 0.95, "mean": 0.15904294065518532, "replicates": 8, "sample_sd": 0.0070181057627164796, "standard_error": 0.002481275087950605 }, "log_loss": { "bootstrap_interval": [ 0.471689163478695, 0.49556318097222296 ], "confidence_level": 0.95, "mean": 0.4834022641639166, "replicates": 8, "sample_sd": 0.018469077998741954, "standard_error": 0.006529805147586853 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1544922656028762, 0.1635930376631673 ], "confidence_level": 0.95, "mean": 0.15901567646822717, "replicates": 8, "sample_sd": 0.007068486600438236, "standard_error": 0.0024990874039480613 }, "log_loss": { "bootstrap_interval": [ 0.47163905791380967, 0.49565703293642444 ], "confidence_level": 0.95, "mean": 0.4833060933590422, "replicates": 8, "sample_sd": 0.018654237181519016, "standard_error": 0.006595268804457162 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.02400423186480739, -0.02179243693387118 ], "confidence_level": 0.95, "mean": -0.02295253502587258, "replicates": 8, "sample_sd": 0.0016841582915956743, "standard_error": 0.000595439874289426 }, "log_loss": { "bootstrap_interval": [ -0.09955569566731325, -0.09034565218434469 ], "confidence_level": 0.95, "mean": -0.09497463315204709, "replicates": 8, "sample_sd": 0.007151164930629009, "standard_error": 0.0025283186079155993 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -5.557081517526281e-05, 1.8678137408288144e-05 ], "confidence_level": 0.95, "mean": -1.1427413904700062e-05, "replicates": 8, "sample_sd": 5.950689724862716e-05, "standard_error": 2.1038865285937683e-05 }, "log_loss": { "bootstrap_interval": [ -0.00026901365546252304, 0.00031294038936998184 ], "confidence_level": 0.95, "mean": 5.2274649916146665e-06, "replicates": 8, "sample_sd": 0.0004237357308308139, "standard_error": 0.00014981320435075306 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -5.557081517526281e-05, 1.8678137408288144e-05 ], "confidence_level": 0.95, "mean": -1.1427413904700062e-05, "replicates": 8, "sample_sd": 5.950689724862716e-05, "standard_error": 2.1038865285937683e-05 }, "log_loss": { "bootstrap_interval": [ -0.00026901365546252304, 0.00031294038936998184 ], "confidence_level": 0.95, "mean": 5.2274649916146665e-06, "replicates": 8, "sample_sd": 0.0004237357308308139, "standard_error": 0.00014981320435075306 } } }, "discount": { "bootstrap_interval": [ 0.9634829922217554, 0.9964760615079313 ], "confidence_level": 0.95, "mean": 0.9819999468733611, "replicates": 8, "sample_sd": 0.026216325955322205, "standard_error": 0.009268870930402613 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.9361190609624468, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07667859410392162, 0.08252311540909804 ], "confidence_level": 0.95, "mean": 0.07961874618503735, "replicates": 8, "sample_sd": 0.004560947187664872, "standard_error": 0.0016125383425157718 }, "log_loss": { "bootstrap_interval": [ 0.2757410625096416, 0.2906968044087382 ], "confidence_level": 0.95, "mean": 0.2832394741026738, "replicates": 8, "sample_sd": 0.011549692806462442, "standard_error": 0.00408343305203554 } }, "independent": { "brier": { "bootstrap_interval": [ 0.05310840015748506, 0.06037405182426801 ], "confidence_level": 0.95, "mean": 0.05667763857306947, "replicates": 8, "sample_sd": 0.005596380434822311, "standard_error": 0.0019786192777812876 }, "log_loss": { "bootstrap_interval": [ 0.17667179512166076, 0.1995871682156029 ], "confidence_level": 0.95, "mean": 0.18825961348563514, "replicates": 8, "sample_sd": 0.01771488578011391, "standard_error": 0.006263157931531844 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10593544976557993, 0.11218360541602533 ], "confidence_level": 0.95, "mean": 0.10914404534872604, "replicates": 8, "sample_sd": 0.004794727973542504, "standard_error": 0.0016951923320183688 }, "log_loss": { "bootstrap_interval": [ 0.3388854022366711, 0.35525124305724304 ], "confidence_level": 0.95, "mean": 0.3469855414655005, "replicates": 8, "sample_sd": 0.012538523242686587, "standard_error": 0.0044330374054844125 } }, "learned": { "brier": { "bootstrap_interval": [ 0.053118294748385704, 0.06030530518444379 ], "confidence_level": 0.95, "mean": 0.05666621115916477, "replicates": 8, "sample_sd": 0.0055588092621464864, "standard_error": 0.0019653358622931847 }, "log_loss": { "bootstrap_interval": [ 0.17708725825743712, 0.19962777155224684 ], "confidence_level": 0.95, "mean": 0.18826484095062673, "replicates": 8, "sample_sd": 0.01745885082708914, "standard_error": 0.006172635905779548 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.05307237606215651, 0.060280428934095756 ], "confidence_level": 0.95, "mean": 0.05667763857306947, "replicates": 8, "sample_sd": 0.005596380434822311, "standard_error": 0.0019786192777812876 }, "log_loss": { "bootstrap_interval": [ 0.17675419952523624, 0.1998622522355637 ], "confidence_level": 0.95, "mean": 0.18825961348563514, "replicates": 8, "sample_sd": 0.01771488578011391, "standard_error": 0.006263157931531844 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.1-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.015468594412342954, -0.012512704298490577 ], "confidence_level": 0.95, "mean": -0.01405903783658526, "replicates": 8, "sample_sd": 0.002316083873159252, "standard_error": 0.0008188593062538553 }, "log_loss": { "bootstrap_interval": [ -0.04550500216149861, -0.03668915617538555 ], "confidence_level": 0.95, "mean": -0.041029801354811005, "replicates": 8, "sample_sd": 0.006840096726770627, "standard_error": 0.0024183393897357084 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.000108897603191771, 0.0003626966737271791 ], "confidence_level": 0.95, "mean": 0.00011208978555755922, "replicates": 8, "sample_sd": 0.00036948878155518777, "standard_error": 0.0001306340115050141 }, "log_loss": { "bootstrap_interval": [ -0.0006777587996285375, 0.0003823650030404308 ], "confidence_level": 0.95, "mean": -0.00015526977125771696, "replicates": 8, "sample_sd": 0.0008110411014012465, "standard_error": 0.00028674633131091387 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.652646259208919e-05, 0.0002697574178651789 ], "confidence_level": 0.95, "mean": 7.736994891608187e-05, "replicates": 8, "sample_sd": 0.00028380422533063834, "standard_error": 0.00010033994613034465 }, "log_loss": { "bootstrap_interval": [ -0.00036557366163285365, 0.0005646853889285814 ], "confidence_level": 0.95, "mean": 7.04663144966422e-05, "replicates": 8, "sample_sd": 0.0007182199701589303, "standard_error": 0.00025392910564148966 } } }, "discount": { "bootstrap_interval": [ 0.9001922102770518, 0.9668603555713031 ], "confidence_level": 0.95, "mean": 0.9354294350367593, "replicates": 8, "sample_sd": 0.05203415597719643, "standard_error": 0.018396852272397058 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.9910231360724382, "discount_min": 0.85188298256698, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17425312690560385, 0.18036282942225854 ], "confidence_level": 0.95, "mean": 0.17714112813499622, "replicates": 8, "sample_sd": 0.004780631918653838, "standard_error": 0.0016902086240184922 }, "log_loss": { "bootstrap_interval": [ 0.5288600477215035, 0.5425496318750024 ], "confidence_level": 0.95, "mean": 0.5354876823415027, "replicates": 8, "sample_sd": 0.01056461491675213, "standard_error": 0.0037351554241299924 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15844484084920746, 0.16781330917876872 ], "confidence_level": 0.95, "mean": 0.1629700005128534, "replicates": 8, "sample_sd": 0.007299913231492227, "standard_error": 0.0025809090740307786 }, "log_loss": { "bootstrap_interval": [ 0.4831745171503684, 0.5061458208344071 ], "confidence_level": 0.95, "mean": 0.4946131507579494, "replicates": 8, "sample_sd": 0.01799071074933011, "standard_error": 0.006360676784608516 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1914202281105597, 0.1987141714145721 ], "confidence_level": 0.95, "mean": 0.1949771317143506, "replicates": 8, "sample_sd": 0.0056748999697914014, "standard_error": 0.002006380125597417 }, "log_loss": { "bootstrap_interval": [ 0.5637895496962125, 0.5808665399411009 ], "confidence_level": 0.95, "mean": 0.5721751720696512, "replicates": 8, "sample_sd": 0.013264221943656803, "standard_error": 0.004689610641761566 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15877780200418717, 0.16783263776942203 ], "confidence_level": 0.95, "mean": 0.16308209029841098, "replicates": 8, "sample_sd": 0.0070597695786954765, "standard_error": 0.0024960054713550335 }, "log_loss": { "bootstrap_interval": [ 0.48349541721173556, 0.5058324003124618 ], "confidence_level": 0.95, "mean": 0.4944578809866917, "replicates": 8, "sample_sd": 0.017251441364528348, "standard_error": 0.00609930558705005 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.15871941991837193, 0.16765724626490838 ], "confidence_level": 0.95, "mean": 0.1630047203494949, "replicates": 8, "sample_sd": 0.0069260501223951665, "standard_error": 0.0024487285041917698 }, "log_loss": { "bootstrap_interval": [ 0.48352996014811994, 0.5052882435940456 ], "confidence_level": 0.95, "mean": 0.49438741467219505, "replicates": 8, "sample_sd": 0.01676759531543335, "standard_error": 0.005928240175867354 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.9090909090909091 ], "oracle_method": "oracle", "population_optimal_scalar": 0.9090909090909091, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.1-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.019858298776139736, -0.017721549714155446 ], "confidence_level": 0.95, "mean": -0.018715097405447842, "replicates": 8, "sample_sd": 0.0016726604319360246, "standard_error": 0.0005913747670221913 }, "log_loss": { "bootstrap_interval": [ -0.0816873696184068, -0.0730789151575956 ], "confidence_level": 0.95, "mean": -0.077203073337515, "replicates": 8, "sample_sd": 0.006503265297629365, "standard_error": 0.0022992514959044373 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0002743610806275262, 4.20369969276758e-05 ], "confidence_level": 0.95, "mean": -0.00011127061346374086, "replicates": 8, "sample_sd": 0.0002445522435223885, "standard_error": 8.646227487453241e-05 }, "log_loss": { "bootstrap_interval": [ -0.0022393112796802, -8.267996200126901e-05 ], "confidence_level": 0.95, "mean": -0.001142980074346666, "replicates": 8, "sample_sd": 0.0016930027974979428, "standard_error": 0.0005985668793392953 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.616705439686586e-05, 0.00015290396609347456 ], "confidence_level": 0.95, "mean": 5.818488885229057e-05, "replicates": 8, "sample_sd": 0.00013850360720349812, "standard_error": 4.896841993619573e-05 }, "log_loss": { "bootstrap_interval": [ -0.00043665257195145597, 0.00028343648577935573 ], "confidence_level": 0.95, "mean": -9.299504550375551e-05, "replicates": 8, "sample_sd": 0.0005622788249430598, "standard_error": 0.00019879558501742058 } } }, "discount": { "bootstrap_interval": [ 0.8759700626222472, 0.9533097433692816 ], "confidence_level": 0.95, "mean": 0.9127459402964009, "replicates": 8, "sample_sd": 0.06044962843803946, "standard_error": 0.021372171094372432 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8421155625489161, "discount_one_count": 2, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07868905580512355, 0.08438785885548519 ], "confidence_level": 0.95, "mean": 0.08145960778074042, "replicates": 8, "sample_sd": 0.004449712808125279, "standard_error": 0.0015732110504790098 }, "log_loss": { "bootstrap_interval": [ 0.2780750164283955, 0.29295118271389936 ], "confidence_level": 0.95, "mean": 0.2854504720970437, "replicates": 8, "sample_sd": 0.011427262778166417, "standard_error": 0.00404014750042105 } }, "independent": { "brier": { "bootstrap_interval": [ 0.059734520770689414, 0.06560911744893755 ], "confidence_level": 0.95, "mean": 0.06285578098875631, "replicates": 8, "sample_sd": 0.004625210911380123, "standard_error": 0.001635258999927448 }, "log_loss": { "bootstrap_interval": [ 0.19881679262305707, 0.21864005592681873 ], "confidence_level": 0.95, "mean": 0.2093903788338754, "replicates": 8, "sample_sd": 0.015481402089691863, "standard_error": 0.005473502199948352 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10504418665959195, 0.11234761933178986 ], "confidence_level": 0.95, "mean": 0.10867614770258291, "replicates": 8, "sample_sd": 0.005710394374144836, "standard_error": 0.0020189292926036623 }, "log_loss": { "bootstrap_interval": [ 0.33467371748605484, 0.3547231852458326 ], "confidence_level": 0.95, "mean": 0.34476458837454727, "replicates": 8, "sample_sd": 0.015212626970181132, "standard_error": 0.0053784758451382204 } }, "learned": { "brier": { "bootstrap_interval": [ 0.05963712462215347, 0.06544516459790617 ], "confidence_level": 0.95, "mean": 0.06274451037529258, "replicates": 8, "sample_sd": 0.004456039210850882, "standard_error": 0.001575447771612905 }, "log_loss": { "bootstrap_interval": [ 0.19853277333830407, 0.21678739805253414 ], "confidence_level": 0.95, "mean": 0.20824739875952872, "replicates": 8, "sample_sd": 0.014179406095662869, "standard_error": 0.005013177101720541 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.05966612955906999, 0.0654525463883592 ], "confidence_level": 0.95, "mean": 0.06268632548644029, "replicates": 8, "sample_sd": 0.004499528681192714, "standard_error": 0.0015908236213073655 }, "log_loss": { "bootstrap_interval": [ 0.19859309150916168, 0.21706672364678098 ], "confidence_level": 0.95, "mean": 0.2083403938050325, "replicates": 8, "sample_sd": 0.01437598557865014, "standard_error": 0.005082678444451764 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.9090909090909091 ], "oracle_method": "oracle", "population_optimal_scalar": 0.9090909090909091, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.25-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.010913349468690926, -0.00810876172004775 ], "confidence_level": 0.95, "mean": -0.009487313860726314, "replicates": 8, "sample_sd": 0.00212036523255305, "standard_error": 0.0007496623172652262 }, "log_loss": { "bootstrap_interval": [ -0.03176154967845568, -0.023705772479460735 ], "confidence_level": 0.95, "mean": -0.027527791080018973, "replicates": 8, "sample_sd": 0.00617155511375792, "standard_error": 0.00218197423570237 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0011522418555593903, -9.293062405734873e-05 ], "confidence_level": 0.95, "mean": -0.0005751441884615099, "replicates": 8, "sample_sd": 0.0008074247917201681, "standard_error": 0.0002854677727617333 }, "log_loss": { "bootstrap_interval": [ -0.005111032288938042, -0.002160307652700532 ], "confidence_level": 0.95, "mean": -0.0036449753553176867, "replicates": 8, "sample_sd": 0.002286943446703258, "standard_error": 0.0008085566096770047 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00021958185976317038, 0.00032559501634774193 ], "confidence_level": 0.95, "mean": 4.4203735301333313e-05, "replicates": 8, "sample_sd": 0.00042403763039103125, "standard_error": 0.00014991994196388652 }, "log_loss": { "bootstrap_interval": [ -0.0006611226879473797, 0.0007661403729567013 ], "confidence_level": 0.95, "mean": 5.6315680810915814e-05, "replicates": 8, "sample_sd": 0.0010789073975580217, "standard_error": 0.00038145136854280374 } } }, "discount": { "bootstrap_interval": [ 0.7732098752404948, 0.8499068146493232 ], "confidence_level": 0.95, "mean": 0.8142648475471204, "replicates": 8, "sample_sd": 0.05989200755677171, "standard_error": 0.021175022341134612 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8682899687775818, "discount_min": 0.7146307800675245, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17647612073618119, 0.1829260473570965 ], "confidence_level": 0.95, "mean": 0.1794838532838206, "replicates": 8, "sample_sd": 0.0051138589693896855, "standard_error": 0.0018080221776435477 }, "log_loss": { "bootstrap_interval": [ 0.5328505072283989, 0.5474281645651422 ], "confidence_level": 0.95, "mean": 0.5398320988006475, "replicates": 8, "sample_sd": 0.011292231493578438, "standard_error": 0.003992406731918805 } }, "independent": { "brier": { "bootstrap_interval": [ 0.16582761829108061, 0.17583899737315015 ], "confidence_level": 0.95, "mean": 0.17057168361155578, "replicates": 8, "sample_sd": 0.007774811739739551, "standard_error": 0.0027488110518093076 }, "log_loss": { "bootstrap_interval": [ 0.5036066426746807, 0.5287536929464149 ], "confidence_level": 0.95, "mean": 0.5159492830759462, "replicates": 8, "sample_sd": 0.019254365426911113, "standard_error": 0.0068074461804063305 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1903617645181104, 0.19877353888871988 ], "confidence_level": 0.95, "mean": 0.1943748998290891, "replicates": 8, "sample_sd": 0.006438871471085747, "standard_error": 0.002276484840196666 }, "log_loss": { "bootstrap_interval": [ 0.5613548225232576, 0.5807653654872382 ], "confidence_level": 0.95, "mean": 0.5710071403004052, "replicates": 8, "sample_sd": 0.014887286419211677, "standard_error": 0.0052634505902454855 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16558356916757563, 0.17474818629602457 ], "confidence_level": 0.95, "mean": 0.16999653942309428, "replicates": 8, "sample_sd": 0.007101250947229132, "standard_error": 0.0025106713498465566 }, "log_loss": { "bootstrap_interval": [ 0.5017191037266852, 0.5234953105712553 ], "confidence_level": 0.95, "mean": 0.5123043077206285, "replicates": 8, "sample_sd": 0.01704549045291074, "standard_error": 0.00602649094395187 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1657422867000591, 0.17450945967988593 ], "confidence_level": 0.95, "mean": 0.16995233568779294, "replicates": 8, "sample_sd": 0.006877871133806434, "standard_error": 0.0024316946594208685 }, "log_loss": { "bootstrap_interval": [ 0.5020020025214664, 0.5229814991215908 ], "confidence_level": 0.95, "mean": 0.5122479920398176, "replicates": 8, "sample_sd": 0.016281082659852463, "standard_error": 0.005756231976920194 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.8 ], "oracle_method": "oracle", "population_optimal_scalar": 0.8, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.25-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.014311139796813402, -0.012492013023596739 ], "confidence_level": 0.95, "mean": -0.013427101954175737, "replicates": 8, "sample_sd": 0.0014129526851712419, "standard_error": 0.0004995542125901631 }, "log_loss": { "bootstrap_interval": [ -0.05812868775430891, -0.05043160773013746 ], "confidence_level": 0.95, "mean": -0.054231508790974055, "replicates": 8, "sample_sd": 0.006086950293768932, "standard_error": 0.0021520619147347296 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0007745247495396443, -0.00025489263794327584 ], "confidence_level": 0.95, "mean": -0.0005156689673459884, "replicates": 8, "sample_sd": 0.00040438051890123895, "standard_error": 0.00014297010354740044 }, "log_loss": { "bootstrap_interval": [ -0.007673818690348881, -0.003694269595737057 ], "confidence_level": 0.95, "mean": -0.005647973853215922, "replicates": 8, "sample_sd": 0.0031243826484577326, "standard_error": 0.0011046360788730238 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.939882074461601e-06, 0.00021931405676067097 ], "confidence_level": 0.95, "mean": 9.414483649488464e-05, "replicates": 8, "sample_sd": 0.0001783998755783207, "standard_error": 6.307388089213344e-05 }, "log_loss": { "bootstrap_interval": [ -0.00027595465029587674, 0.0008713762138003803 ], "confidence_level": 0.95, "mean": 0.00019239874935372478, "replicates": 8, "sample_sd": 0.0009267516798837053, "standard_error": 0.00032765619866089623 } } }, "discount": { "bootstrap_interval": [ 0.7614519585473556, 0.841179879873069 ], "confidence_level": 0.95, "mean": 0.802270060417436, "replicates": 8, "sample_sd": 0.0614535737501734, "standard_error": 0.02172711936344761 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8951930908642205, "discount_min": 0.7146415307635985, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08202986765572638, 0.08822834334462916 ], "confidence_level": 0.95, "mean": 0.08496728187961188, "replicates": 8, "sample_sd": 0.004856302975269613, "standard_error": 0.0017169623826547749 }, "log_loss": { "bootstrap_interval": [ 0.28387228750937277, 0.3000405203187463 ], "confidence_level": 0.95, "mean": 0.29151875529082766, "replicates": 8, "sample_sd": 0.012352799114050224, "standard_error": 0.0043673740100900445 } }, "independent": { "brier": { "bootstrap_interval": [ 0.06848615190636567, 0.07549977811995669 ], "confidence_level": 0.95, "mean": 0.07205584889278213, "replicates": 8, "sample_sd": 0.005484615764094766, "standard_error": 0.0019391044994970233 }, "log_loss": { "bootstrap_interval": [ 0.23137835275448318, 0.2534535965848561 ], "confidence_level": 0.95, "mean": 0.24293522035306953, "replicates": 8, "sample_sd": 0.017167990516192146, "standard_error": 0.0060698012566729005 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10445215772319595, 0.11229748975008268 ], "confidence_level": 0.95, "mean": 0.10829839718771324, "replicates": 8, "sample_sd": 0.006040941932025175, "standard_error": 0.0021357955024445823 }, "log_loss": { "bootstrap_interval": [ 0.3334455636550256, 0.3545014760987462 ], "confidence_level": 0.95, "mean": 0.3440690215177385, "replicates": 8, "sample_sd": 0.016362456544759204, "standard_error": 0.005785001989834719 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0680961301750588, 0.07470751909247221 ], "confidence_level": 0.95, "mean": 0.07154017992543614, "replicates": 8, "sample_sd": 0.005120311961594283, "standard_error": 0.0018103036549169552 }, "log_loss": { "bootstrap_interval": [ 0.22730950384931786, 0.24610795602142146 ], "confidence_level": 0.95, "mean": 0.23728724649985358, "replicates": 8, "sample_sd": 0.014663518016646126, "standard_error": 0.005184336512810794 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06813764642705442, 0.0746326898841535 ], "confidence_level": 0.95, "mean": 0.07144603508894126, "replicates": 8, "sample_sd": 0.0050522727893388365, "standard_error": 0.0017862481748728823 }, "log_loss": { "bootstrap_interval": [ 0.22720094090143672, 0.2460821603954735 ], "confidence_level": 0.95, "mean": 0.23709484775049985, "replicates": 8, "sample_sd": 0.014527682929570636, "standard_error": 0.005136311557213722 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.8 ], "oracle_method": "oracle", "population_optimal_scalar": 0.8, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.5-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.005393696449314328, -0.0032213743213946947 ], "confidence_level": 0.95, "mean": -0.004202948921495524, "replicates": 8, "sample_sd": 0.0016790656057058956, "standard_error": 0.0005936393379258682 }, "log_loss": { "bootstrap_interval": [ -0.015345356325629294, -0.0091108812740186 ], "confidence_level": 0.95, "mean": -0.012006807160640537, "replicates": 8, "sample_sd": 0.00486689481573159, "standard_error": 0.0017207071637627299 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0037722546914884976, -0.002031246832685449 ], "confidence_level": 0.95, "mean": -0.002878493235080881, "replicates": 8, "sample_sd": 0.0013495445414950895, "standard_error": 0.0004771360484022339 }, "log_loss": { "bootstrap_interval": [ -0.018586380678720413, -0.011811063912360253 ], "confidence_level": 0.95, "mean": -0.015265626050923811, "replicates": 8, "sample_sd": 0.005258182717234976, "standard_error": 0.001859048328037379 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.0003039211931934027, 0.0003746412997236543 ], "confidence_level": 0.95, "mean": 3.669540664580709e-05, "replicates": 8, "sample_sd": 0.000523943667016641, "standard_error": 0.00018524205995360665 }, "log_loss": { "bootstrap_interval": [ -0.0007628346419778524, 0.0010615450713781607 ], "confidence_level": 0.95, "mean": 0.0001787647280008242, "replicates": 8, "sample_sd": 0.0014326296934996153, "standard_error": 0.0005065110856013915 } } }, "discount": { "bootstrap_interval": [ 0.6290980501352931, 0.7087038735996015 ], "confidence_level": 0.95, "mean": 0.6710673520157847, "replicates": 8, "sample_sd": 0.06115399875988565, "standard_error": 0.02162120360989443 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.7316013621321054, "discount_min": 0.5702709578427307, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18058817332348254, 0.18787478464257656 ], "confidence_level": 0.95, "mean": 0.1839425750533926, "replicates": 8, "sample_sd": 0.0056478398824274054, "standard_error": 0.0019968129399601255 }, "log_loss": { "bootstrap_interval": [ 0.541094182798983, 0.5573838631558742 ], "confidence_level": 0.95, "mean": 0.5486048271050566, "replicates": 8, "sample_sd": 0.012584192572757441, "standard_error": 0.004449183951977086 } }, "independent": { "brier": { "bootstrap_interval": [ 0.17754609490075585, 0.1882153105949708 ], "confidence_level": 0.95, "mean": 0.18261811936697794, "replicates": 8, "sample_sd": 0.00830103148327283, "standard_error": 0.002934857826332621 }, "log_loss": { "bootstrap_interval": [ 0.5380826610093806, 0.5664611160506442 ], "confidence_level": 0.95, "mean": 0.5518636459953399, "replicates": 8, "sample_sd": 0.021898800675118863, "standard_error": 0.007742395228614546 } }, "individual": { "brier": { "bootstrap_interval": [ 0.18950572239035263, 0.19863249929560858 ], "confidence_level": 0.95, "mean": 0.19388169492360882, "replicates": 8, "sample_sd": 0.0071470337450813765, "standard_error": 0.002526858013258064 }, "log_loss": { "bootstrap_interval": [ 0.5595474700780493, 0.5809763353065956 ], "confidence_level": 0.95, "mean": 0.5701575733519997, "replicates": 8, "sample_sd": 0.016369939259517546, "standard_error": 0.0057876475290083735 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17534642024142597, 0.18445599814092664 ], "confidence_level": 0.95, "mean": 0.17973962613189706, "replicates": 8, "sample_sd": 0.007090095266830508, "standard_error": 0.002506727221217248 }, "log_loss": { "bootstrap_interval": [ 0.5259450027108502, 0.5477560889257161 ], "confidence_level": 0.95, "mean": 0.5365980199444161, "replicates": 8, "sample_sd": 0.01692232665834029, "standard_error": 0.0059829459667831535 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1754542413674122, 0.18421566747344945 ], "confidence_level": 0.95, "mean": 0.17970293072525126, "replicates": 8, "sample_sd": 0.006773315230253962, "standard_error": 0.002394728565213349 }, "log_loss": { "bootstrap_interval": [ 0.5266360738226472, 0.5469724045725538 ], "confidence_level": 0.95, "mean": 0.5364192552164153, "replicates": 8, "sample_sd": 0.01590685010992015, "standard_error": 0.005623920790021258 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.6666666666666666 ], "oracle_method": "oracle", "population_optimal_scalar": 0.6666666666666666, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.5-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.007123177859149495, -0.005651356915561963 ], "confidence_level": 0.95, "mean": -0.006393179033024209, "replicates": 8, "sample_sd": 0.001144870342074192, "standard_error": 0.0004047727912300117 }, "log_loss": { "bootstrap_interval": [ -0.028207341745673888, -0.021956065151749967 ], "confidence_level": 0.95, "mean": -0.02518170547404871, "replicates": 8, "sample_sd": 0.004881311644015502, "standard_error": 0.0017258042822841078 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.002849387255577932, -0.0012877585641154698 ], "confidence_level": 0.95, "mean": -0.002041864593292036, "replicates": 8, "sample_sd": 0.0012519013581251649, "standard_error": 0.0004426139698534763 }, "log_loss": { "bootstrap_interval": [ -0.024360202072205656, -0.016722832064784095 ], "confidence_level": 0.95, "mean": -0.02038968684645933, "replicates": 8, "sample_sd": 0.005912454670173854, "standard_error": 0.002090368395369002 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -7.965329567018176e-05, 0.00016834774140811153 ], "confidence_level": 0.95, "mean": 3.745763924755188e-05, "replicates": 8, "sample_sd": 0.00019088183100608686, "standard_error": 6.74869185548543e-05 }, "log_loss": { "bootstrap_interval": [ -0.00017074752829916023, 0.0012886989843201371 ], "confidence_level": 0.95, "mean": 0.0005117326460168659, "replicates": 8, "sample_sd": 0.001124924649037254, "standard_error": 0.0003977209238290697 } } }, "discount": { "bootstrap_interval": [ 0.6237666005176268, 0.6994452600214871 ], "confidence_level": 0.95, "mean": 0.6616426526401765, "replicates": 8, "sample_sd": 0.0584530185962845, "standard_error": 0.02066626291512807 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.7490398907370075, "discount_min": 0.5752016722405258, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0879748180436677, 0.09512339501805384 ], "confidence_level": 0.95, "mean": 0.09142629658016359, "replicates": 8, "sample_sd": 0.005512617776567518, "standard_error": 0.0019490047059501998 }, "log_loss": { "bootstrap_interval": [ 0.2958075654274668, 0.31385457115983006 ], "confidence_level": 0.95, "mean": 0.30438543890497605, "replicates": 8, "sample_sd": 0.01405553924124435, "standard_error": 0.00496938355535875 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08243832315352989, 0.09167955201116279 ], "confidence_level": 0.95, "mean": 0.08707498214043141, "replicates": 8, "sample_sd": 0.00719046563390761, "standard_error": 0.0025422135048124486 }, "log_loss": { "bootstrap_interval": [ 0.28576058329566995, 0.31354470936579704 ], "confidence_level": 0.95, "mean": 0.29959342027738667, "replicates": 8, "sample_sd": 0.021769666224218876, "standard_error": 0.007696739305656455 } }, "individual": { "brier": { "bootstrap_interval": [ 0.103595756228063, 0.11168686177748238 ], "confidence_level": 0.95, "mean": 0.10774614617706736, "replicates": 8, "sample_sd": 0.006332313117656662, "standard_error": 0.002238810773045777 }, "log_loss": { "bootstrap_interval": [ 0.3322866640657971, 0.35475333927696284 ], "confidence_level": 0.95, "mean": 0.3436452340089625, "replicates": 8, "sample_sd": 0.01747682052815941, "standard_error": 0.0061789891545208885 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08118798945487402, 0.08895346668576962 ], "confidence_level": 0.95, "mean": 0.08503311754713938, "replicates": 8, "sample_sd": 0.0060813747043352235, "standard_error": 0.002150090646185886 }, "log_loss": { "bootstrap_interval": [ 0.26856595692474977, 0.2896035024422186 ], "confidence_level": 0.95, "mean": 0.27920373343092736, "replicates": 8, "sample_sd": 0.016341251502465168, "standard_error": 0.005777504875233989 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08107333692757315, 0.08889819879757942 ], "confidence_level": 0.95, "mean": 0.08499565990789182, "replicates": 8, "sample_sd": 0.005986480777078366, "standard_error": 0.0021165405764575125 }, "log_loss": { "bootstrap_interval": [ 0.26828529176879357, 0.2888976284386162 ], "confidence_level": 0.95, "mean": 0.27869200078491047, "replicates": 8, "sample_sd": 0.015892938731512533, "standard_error": 0.00561900237501742 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.6666666666666666 ], "oracle_method": "oracle", "population_optimal_scalar": 0.6666666666666666, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.75-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0019202675730518259, -0.0005296122320127411 ], "confidence_level": 0.95, "mean": -0.0011689374394721896, "replicates": 8, "sample_sd": 0.0010926224908894536, "standard_error": 0.00038630038629243463 }, "log_loss": { "bootstrap_interval": [ -0.005351078037689201, -0.0013762560360224905 ], "confidence_level": 0.95, "mean": -0.003220742537905663, "replicates": 8, "sample_sd": 0.0031053555687288975, "standard_error": 0.0010979089903218056 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.007515543006033326, -0.004891571076493524 ], "confidence_level": 0.95, "mean": -0.006214109895671616, "replicates": 8, "sample_sd": 0.0019956127965590365, "standard_error": 0.0007055556705347723 }, "log_loss": { "bootstrap_interval": [ -0.03744763135934648, -0.026379350027418688 ], "confidence_level": 0.95, "mean": -0.03188724349648628, "replicates": 8, "sample_sd": 0.008368838493512658, "standard_error": 0.0029588312247089054 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.0002905575442265976, 0.00039888225018473375 ], "confidence_level": 0.95, "mean": 5.6278538888523966e-05, "replicates": 8, "sample_sd": 0.0005346765841891999, "standard_error": 0.00018903671921092162 }, "log_loss": { "bootstrap_interval": [ -0.0006927098691406672, 0.0012147622791001678 ], "confidence_level": 0.95, "mean": 0.00029289471981450155, "replicates": 8, "sample_sd": 0.0015133002824114935, "standard_error": 0.0005350324458323422 } } }, "discount": { "bootstrap_interval": [ 0.5339846565964286, 0.6052960950215791 ], "confidence_level": 0.95, "mean": 0.5710985182545075, "replicates": 8, "sample_sd": 0.05525529242344963, "standard_error": 0.019535695984533447 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.6295857681399497, "discount_min": 0.47988630625563045, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18528586894987115, 0.1931402634300129 ], "confidence_level": 0.95, "mean": 0.18887820328289345, "replicates": 8, "sample_sd": 0.006126244595032101, "standard_error": 0.0021659545481773164 }, "log_loss": { "bootstrap_interval": [ 0.5506744359602668, 0.5685614442049406 ], "confidence_level": 0.95, "mean": 0.5588590745470476, "replicates": 8, "sample_sd": 0.013861039161481169, "standard_error": 0.004900617392687815 } }, "independent": { "brier": { "bootstrap_interval": [ 0.18839296724390153, 0.1998589078905045 ], "confidence_level": 0.95, "mean": 0.19392337573909288, "replicates": 8, "sample_sd": 0.00888062347826209, "standard_error": 0.003139774541321794 }, "log_loss": { "bootstrap_interval": [ 0.5721081532384246, 0.6037767787107865 ], "confidence_level": 0.95, "mean": 0.5875255755056282, "replicates": 8, "sample_sd": 0.024496105762540427, "standard_error": 0.0086606812486776 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1893792600975036, 0.1988278210313756 ], "confidence_level": 0.95, "mean": 0.19386376085344048, "replicates": 8, "sample_sd": 0.007383401697858005, "standard_error": 0.002610426704389832 }, "log_loss": { "bootstrap_interval": [ 0.5597223612349367, 0.5813716502240927 ], "confidence_level": 0.95, "mean": 0.5700788344611047, "replicates": 8, "sample_sd": 0.016928713781026974, "standard_error": 0.005985204155665166 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18336292276202798, 0.1925512572307019 ], "confidence_level": 0.95, "mean": 0.18770926584342126, "replicates": 8, "sample_sd": 0.007017047687163071, "standard_error": 0.002480901001751193 }, "log_loss": { "bootstrap_interval": [ 0.5456444051553194, 0.5671006498522181 ], "confidence_level": 0.95, "mean": 0.5556383320091419, "replicates": 8, "sample_sd": 0.016493782874399742, "standard_error": 0.005831432858953301 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18367143741046127, 0.19234731113872955 ], "confidence_level": 0.95, "mean": 0.18765298730453273, "replicates": 8, "sample_sd": 0.006689016016515177, "standard_error": 0.0023649242923716542 }, "log_loss": { "bootstrap_interval": [ 0.5460812325346442, 0.5661574255857513 ], "confidence_level": 0.95, "mean": 0.5553454372893274, "replicates": 8, "sample_sd": 0.015545340438216378, "standard_error": 0.0054961078198581285 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5714285714285714 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5714285714285714, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho0.75-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.002329770697753186, -0.00128828690401157 ], "confidence_level": 0.95, "mean": -0.001814058290596044, "replicates": 8, "sample_sd": 0.0008072159574415658, "standard_error": 0.00028539393869446135 }, "log_loss": { "bootstrap_interval": [ -0.0087994844777723, -0.004648447254540793 ], "confidence_level": 0.95, "mean": -0.006722152617227799, "replicates": 8, "sample_sd": 0.0031991748438501624, "standard_error": 0.001131079113143932 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.005549294985852981, -0.0029173551352960955 ], "confidence_level": 0.95, "mean": -0.0041544392531593, "replicates": 8, "sample_sd": 0.0020811533333449255, "standard_error": 0.0007357988173485921 }, "log_loss": { "bootstrap_interval": [ -0.04773955212200865, -0.03599259732021243 ], "confidence_level": 0.95, "mean": -0.04184882627593212, "replicates": 8, "sample_sd": 0.009069797508209975, "standard_error": 0.003206657661022062 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00010377245348239729, 0.00018759423327349364 ], "confidence_level": 0.95, "mean": 4.165561602128762e-05, "replicates": 8, "sample_sd": 0.00022671718224513633, "standard_error": 8.015662848852111e-05 }, "log_loss": { "bootstrap_interval": [ -7.305305527099608e-05, 0.0014337267717918142 ], "confidence_level": 0.95, "mean": 0.0006838228813220507, "replicates": 8, "sample_sd": 0.0011578543119642286, "standard_error": 0.00040936331780799513 } } }, "discount": { "bootstrap_interval": [ 0.5312425668780633, 0.5985514587734581 ], "confidence_level": 0.95, "mean": 0.5650758198222691, "replicates": 8, "sample_sd": 0.051837481891406, "standard_error": 0.01832731748252402 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.6387679297725659, "discount_min": 0.48779003911455054, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09515584180935721, 0.10289546553015014 ], "confidence_level": 0.95, "mean": 0.09891769104886339, "replicates": 8, "sample_sd": 0.005972200217173526, "standard_error": 0.002111491636083586 }, "log_loss": { "bootstrap_interval": [ 0.31162943283726297, 0.33133831280523646 ], "confidence_level": 0.95, "mean": 0.32124480362108965, "replicates": 8, "sample_sd": 0.015497967291205695, "standard_error": 0.005479358883109428 } }, "independent": { "brier": { "bootstrap_interval": [ 0.09593674667799143, 0.10665040168233417 ], "confidence_level": 0.95, "mean": 0.10125807201142664, "replicates": 8, "sample_sd": 0.008229106653027564, "standard_error": 0.002909428558731562 }, "log_loss": { "bootstrap_interval": [ 0.3398055891656848, 0.37329915991562096 ], "confidence_level": 0.95, "mean": 0.356371477279794, "replicates": 8, "sample_sd": 0.02584100321749062, "standard_error": 0.009136174303875505 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10337793940722322, 0.11181321195020469 ], "confidence_level": 0.95, "mean": 0.10749807007090667, "replicates": 8, "sample_sd": 0.006495836244050999, "standard_error": 0.0022966249288229068 }, "log_loss": { "bootstrap_interval": [ 0.33185788264390303, 0.3552523520346478 ], "confidence_level": 0.95, "mean": 0.3436335678809739, "replicates": 8, "sample_sd": 0.01819648125975606, "standard_error": 0.00643342764625372 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09300867238971439, 0.10143245951715216 ], "confidence_level": 0.95, "mean": 0.09710363275826735, "replicates": 8, "sample_sd": 0.0064846441056856515, "standard_error": 0.0022926679103558493 }, "log_loss": { "bootstrap_interval": [ 0.3034303543301118, 0.3256890664775087 ], "confidence_level": 0.95, "mean": 0.31452265100386184, "replicates": 8, "sample_sd": 0.01718946071235163, "standard_error": 0.006077392117321789 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.09302185670998046, 0.1012860434336442 ], "confidence_level": 0.95, "mean": 0.09706197714224606, "replicates": 8, "sample_sd": 0.006382613390531473, "standard_error": 0.002256594605068433 }, "log_loss": { "bootstrap_interval": [ 0.30332550605398834, 0.3247823272941491 ], "confidence_level": 0.95, "mean": 0.3138388281225398, "replicates": 8, "sample_sd": 0.016693292185549994, "standard_error": 0.005901970052365401 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5714285714285714 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5714285714285714, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho1-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.00016364122072196578, 0.00027284054575019655 ], "confidence_level": 0.95, "mean": 5.417565099974375e-05, "replicates": 8, "sample_sd": 0.0003374385605486479, "standard_error": 0.00011930254719888816 }, "log_loss": { "bootstrap_interval": [ -0.000420038808850744, 0.0010243736522804006 ], "confidence_level": 0.95, "mean": 0.00031282080976346993, "replicates": 8, "sample_sd": 0.0011149038923301022, "standard_error": 0.0003941780513189458 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.013177976460348996, -0.008872729819228747 ], "confidence_level": 0.95, "mean": -0.010884135800468646, "replicates": 8, "sample_sd": 0.0033428581669544817, "standard_error": 0.001181878839199173 }, "log_loss": { "bootstrap_interval": [ -0.06095781537293242, -0.0460628394521343 ], "confidence_level": 0.95, "mean": -0.05278576558323167, "replicates": 8, "sample_sd": 0.011677508176619932, "standard_error": 0.004128622609524655 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.0001636412207219657, 0.0002728405457502136 ], "confidence_level": 0.95, "mean": 5.417565099974722e-05, "replicates": 8, "sample_sd": 0.00033743856054865254, "standard_error": 0.0001193025471988898 }, "log_loss": { "bootstrap_interval": [ -0.0004200388088507579, 0.0010243736522803997 ], "confidence_level": 0.95, "mean": 0.00031282080976345605, "replicates": 8, "sample_sd": 0.0011149038923301054, "standard_error": 0.00039417805131894694 } } }, "discount": { "bootstrap_interval": [ 0.46846635927319635, 0.5217140371215196 ], "confidence_level": 0.95, "mean": 0.4957637618343603, "replicates": 8, "sample_sd": 0.04153988128480036, "standard_error": 0.014686565873083243 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.5453332151779846, "discount_min": 0.4273494987051204, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19115545613020565, 0.19958473506696908 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5631282851734062, 0.5823252274371586 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348748, "standard_error": 0.005382364421255804 } }, "independent": { "brier": { "bootstrap_interval": [ 0.20028877108434207, 0.21287942584847006 ], "confidence_level": 0.95, "mean": 0.2059093286797352, "replicates": 8, "sample_sd": 0.009873668119329283, "standard_error": 0.0034908688411815804 }, "log_loss": { "bootstrap_interval": [ 0.6094005564015583, 0.6439421856651459 ], "confidence_level": 0.95, "mean": 0.6249939088267681, "replicates": 8, "sample_sd": 0.026958293387921, "standard_error": 0.009531196031907701 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19109830400471042, 0.1995570185926899 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.56330495255794, 0.582935894267488 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19114893603059935, 0.19985096408878814 ], "confidence_level": 0.95, "mean": 0.19502519287926653, "replicates": 8, "sample_sd": 0.006700670866674529, "standard_error": 0.00236904490416235 }, "log_loss": { "bootstrap_interval": [ 0.5631744746415268, 0.5829606957863118 ], "confidence_level": 0.95, "mean": 0.5722081432435364, "replicates": 8, "sample_sd": 0.0154746166590676, "standard_error": 0.005471103187944508 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19116949730067523, 0.19951132124383997 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5630470753630746, 0.5828806871758583 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n2-rho1-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -8.570703380609071e-05, 0.00017543303982253677 ], "confidence_level": 0.95, "mean": 3.2815773903000986e-05, "replicates": 8, "sample_sd": 0.0002081477777885751, "standard_error": 7.359135258160605e-05 }, "log_loss": { "bootstrap_interval": [ -0.000156049604436137, 0.0011448172221134396 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.008064383588006293, -0.0056358482219784085 ], "confidence_level": 0.95, "mean": -0.006881649961832515, "replicates": 8, "sample_sd": 0.0018575792689995401, "standard_error": 0.0006567534488505623 }, "log_loss": { "bootstrap_interval": [ -0.07477159898978382, -0.06152938646076538 ], "confidence_level": 0.95, "mean": -0.06772125513896934, "replicates": 8, "sample_sd": 0.01025657551418292, "standard_error": 0.0036262470489153214 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -8.570703380609071e-05, 0.00017543303982253677 ], "confidence_level": 0.95, "mean": 3.2815773903000986e-05, "replicates": 8, "sample_sd": 0.0002081477777885751, "standard_error": 7.359135258160605e-05 }, "log_loss": { "bootstrap_interval": [ -0.000156049604436137, 0.0011448172221134396 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } } }, "discount": { "bootstrap_interval": [ 0.47364467357646656, 0.5178250534866158 ], "confidence_level": 0.95, "mean": 0.49593841193576715, "replicates": 8, "sample_sd": 0.034948970894948285, "standard_error": 0.012356327157654607 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.5397081015060035, "discount_min": 0.4394751545183566, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.10485809182433103, 0.11280075032555845 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.33482255855878673, 0.3565294025886357 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "independent": { "brier": { "bootstrap_interval": [ 0.11059057872297558, 0.12065531469572621 ], "confidence_level": 0.95, "mean": 0.11528294409834614, "replicates": 8, "sample_sd": 0.00784957155594904, "standard_error": 0.0027752426383103024 }, "log_loss": { "bootstrap_interval": [ 0.396717225370224, 0.43128569855642507 ], "confidence_level": 0.95, "mean": 0.41303095293164493, "replicates": 8, "sample_sd": 0.02676198338226836, "standard_error": 0.009461789963801827 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10483686076928903, 0.11288572970935466 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3348663357232349, 0.3570510602273047 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10466396457975613, 0.11279870317757779 ], "confidence_level": 0.95, "mean": 0.10840129413651362, "replicates": 8, "sample_sd": 0.006334716340172474, "standard_error": 0.0022396604405145923 }, "log_loss": { "bootstrap_interval": [ 0.3351317727003469, 0.3572172137598196 ], "confidence_level": 0.95, "mean": 0.3453096977926756, "replicates": 8, "sample_sd": 0.017009453256656443, "standard_error": 0.006013749871028687 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1047570223262377, 0.11275967041149533 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.33508958912300857, 0.356751074408452 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.15156011200512032, -0.14970430125917408 ], "confidence_level": 0.95, "mean": -0.15063779940623567, "replicates": 8, "sample_sd": 0.0014209049727733774, "standard_error": 0.0005023657708348709 }, "log_loss": { "bootstrap_interval": [ -0.4782532494864198, -0.46967128292257787 ], "confidence_level": 0.95, "mean": -0.4738308564649181, "replicates": 8, "sample_sd": 0.006670234488603748, "standard_error": 0.0023582840194980464 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -3.455166260711616e-05, 1.4681467501259388e-05 ], "confidence_level": 0.95, "mean": -1.0604204030659767e-05, "replicates": 8, "sample_sd": 3.7911891403291205e-05, "standard_error": 1.3403877749437592e-05 }, "log_loss": { "bootstrap_interval": [ -3.6583728615539185e-05, 0.00017277543969750108 ], "confidence_level": 0.95, "mean": 4.951840499310819e-05, "replicates": 8, "sample_sd": 0.00016465044126053114, "standard_error": 5.821272177033944e-05 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.455166260711616e-05, 1.4681467501259388e-05 ], "confidence_level": 0.95, "mean": -1.0604204030659767e-05, "replicates": 8, "sample_sd": 3.7911891403291205e-05, "standard_error": 1.3403877749437592e-05 }, "log_loss": { "bootstrap_interval": [ -3.6583728615539185e-05, 0.00017277543969750108 ], "confidence_level": 0.95, "mean": 4.951840499310819e-05, "replicates": 8, "sample_sd": 0.00016465044126053114, "standard_error": 5.821272177033944e-05 } } }, "discount": { "bootstrap_interval": [ 0.9451807527669196, 0.9922304021968322 ], "confidence_level": 0.95, "mean": 0.9702649776965915, "replicates": 8, "sample_sd": 0.03646859066186254, "standard_error": 0.012893593878659702 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.9193881319059962, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1602624238580483, 0.1616959833676296 ], "confidence_level": 0.95, "mean": 0.1609613102197546, "replicates": 8, "sample_sd": 0.0011023343947350846, "standard_error": 0.0003897340628261734 }, "log_loss": { "bootstrap_interval": [ 0.5099922881630345, 0.512969077198491 ], "confidence_level": 0.95, "mean": 0.5114555199972123, "replicates": 8, "sample_sd": 0.002304973413985085, "standard_error": 0.0008149311657417804 } }, "independent": { "brier": { "bootstrap_interval": [ 0.008983969197615812, 0.011797480261452286 ], "confidence_level": 0.95, "mean": 0.010334115017549593, "replicates": 8, "sample_sd": 0.0021652071270071922, "standard_error": 0.0007655163210901139 }, "log_loss": { "bootstrap_interval": [ 0.03218848477430051, 0.042937900340119475 ], "confidence_level": 0.95, "mean": 0.03757514512730103, "replicates": 8, "sample_sd": 0.00825572376324761, "standard_error": 0.002918839128297654 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19328585817404292, 0.19910773781561966 ], "confidence_level": 0.95, "mean": 0.19616138579072515, "replicates": 8, "sample_sd": 0.004520943891208201, "standard_error": 0.001598395041418608 }, "log_loss": { "bootstrap_interval": [ 0.5684066137180377, 0.5817573200528643 ], "confidence_level": 0.95, "mean": 0.5749023486997822, "replicates": 8, "sample_sd": 0.010327742370136919, "standard_error": 0.0036514083321357205 } }, "learned": { "brier": { "bootstrap_interval": [ 0.008991239728582692, 0.011778345032097381 ], "confidence_level": 0.95, "mean": 0.010323510813518932, "replicates": 8, "sample_sd": 0.0021519855988576396, "standard_error": 0.0007608418049840152 }, "log_loss": { "bootstrap_interval": [ 0.032385703913198095, 0.042994754966559856 ], "confidence_level": 0.95, "mean": 0.037624663532294135, "replicates": 8, "sample_sd": 0.00818163283618966, "standard_error": 0.002892644029824117 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.008994490442252255, 0.011761882821736822 ], "confidence_level": 0.95, "mean": 0.010334115017549593, "replicates": 8, "sample_sd": 0.0021652071270071922, "standard_error": 0.0007655163210901139 }, "log_loss": { "bootstrap_interval": [ 0.0321404793878026, 0.042815998501351454 ], "confidence_level": 0.95, "mean": 0.03757514512730103, "replicates": 8, "sample_sd": 0.00825572376324761, "standard_error": 0.002918839128297654 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.054583205786538774, -0.053496426271111064 ], "confidence_level": 0.95, "mean": -0.05402127330750593, "replicates": 8, "sample_sd": 0.0008404993349573839, "standard_error": 0.00029716138966557476 }, "log_loss": { "bootstrap_interval": [ -0.2599242509145047, -0.25695805218736606 ], "confidence_level": 0.95, "mean": -0.25843595544047204, "replicates": 8, "sample_sd": 0.0022640258808550497, "standard_error": 0.000800454026567226 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "discount": { "bootstrap_interval": [ 1.0, 1.0 ], "confidence_level": 0.95, "mean": 1.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 1.0, "discount_one_count": 8, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05348775991829462, 0.054567613018949565 ], "confidence_level": 0.95, "mean": 0.05402127333043332, "replicates": 8, "sample_sd": 0.000840499364792614, "standard_error": 0.0002971614002139215 }, "log_loss": { "bootstrap_interval": [ 0.2569835650661224, 0.25992534667292977 ], "confidence_level": 0.95, "mean": 0.2584360684473848, "replicates": 8, "sample_sd": 0.0022641108312168924, "standard_error": 0.0008004840610556876 } }, "independent": { "brier": { "bootstrap_interval": [ 7.403924883674551e-13, 6.2289181354118e-11 ], "confidence_level": 0.95, "mean": 2.2927390717155637e-11, "replicates": 8, "sample_sd": 5.4789384473201153e-11, "standard_error": 1.9370972649018735e-11 }, "log_loss": { "bootstrap_interval": [ 1.5893620190494412e-08, 2.5709882908302387e-07 ], "confidence_level": 0.95, "mean": 1.1300691277030559e-07, "replicates": 8, "sample_sd": 1.9742208131804873e-07, "standard_error": 6.979924622797714e-08 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1060213568860606, 0.11218079120943264 ], "confidence_level": 0.95, "mean": 0.10914404534872604, "replicates": 8, "sample_sd": 0.004794727973542504, "standard_error": 0.0016951923320183688 }, "log_loss": { "bootstrap_interval": [ 0.3388184096210745, 0.3548884733338829 ], "confidence_level": 0.95, "mean": 0.3469855414655005, "replicates": 8, "sample_sd": 0.012538523242686587, "standard_error": 0.0044330374054844125 } }, "learned": { "brier": { "bootstrap_interval": [ 1.0513589324478222e-13, 6.228929589800899e-11 ], "confidence_level": 0.95, "mean": 2.2927390717155637e-11, "replicates": 8, "sample_sd": 5.4789384473201153e-11, "standard_error": 1.9370972649018735e-11 }, "log_loss": { "bootstrap_interval": [ 1.5976376565685853e-08, 2.6279005708939414e-07 ], "confidence_level": 0.95, "mean": 1.1300691277030559e-07, "replicates": 8, "sample_sd": 1.9742208131804873e-07, "standard_error": 6.979924622797714e-08 } }, "oracle": { "brier": { "bootstrap_interval": [ 7.396459062393552e-13, 6.228941337892282e-11 ], "confidence_level": 0.95, "mean": 2.2927390717155637e-11, "replicates": 8, "sample_sd": 5.4789384473201153e-11, "standard_error": 1.9370972649018735e-11 }, "log_loss": { "bootstrap_interval": [ 1.452969929580232e-08, 2.5489528384088913e-07 ], "confidence_level": 0.95, "mean": 1.1300691277030559e-07, "replicates": 8, "sample_sd": 1.9742208131804873e-07, "standard_error": 6.979924622797714e-08 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.1-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.098415547124443, -0.09341470855099764 ], "confidence_level": 0.95, "mean": -0.09599311295221474, "replicates": 8, "sample_sd": 0.003851548090499968, "standard_error": 0.0013617278864293128 }, "log_loss": { "bootstrap_interval": [ -0.2998770894809674, -0.28115814893342567 ], "confidence_level": 0.95, "mean": -0.2910276558326176, "replicates": 8, "sample_sd": 0.014470244743532175, "standard_error": 0.005116004091790297 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.008987225227392774, -0.00791609357199859 ], "confidence_level": 0.95, "mean": -0.008479803840371482, "replicates": 8, "sample_sd": 0.0008320498530032058, "standard_error": 0.0002941740466719184 }, "log_loss": { "bootstrap_interval": [ -0.16313794416926677, -0.12337720790738554 ], "confidence_level": 0.95, "mean": -0.1418113797545293, "replicates": 8, "sample_sd": 0.03088163181585429, "standard_error": 0.010918305635548402 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.9939697422377827e-06, 7.357692619344869e-05 ], "confidence_level": 0.95, "mean": 3.5592301666632735e-05, "replicates": 8, "sample_sd": 5.831615377602786e-05, "standard_error": 2.0617873893873396e-05 }, "log_loss": { "bootstrap_interval": [ -0.0006079646536230239, 0.0003615962051540314 ], "confidence_level": 0.95, "mean": -8.468695567703105e-05, "replicates": 8, "sample_sd": 0.0007489049086457234, "standard_error": 0.00026477786968364146 } } }, "discount": { "bootstrap_interval": [ 0.33214884267053624, 0.3610337721690665 ], "confidence_level": 0.95, "mean": 0.3473893114165357, "replicates": 8, "sample_sd": 0.022445435376664855, "standard_error": 0.007935659780762074 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3767866478320617, "discount_min": 0.30579550426998003, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16220702799076936, 0.16461669115247804 ], "confidence_level": 0.95, "mean": 0.1634056787929699, "replicates": 8, "sample_sd": 0.0018486330356592558, "standard_error": 0.0006535904777200662 }, "log_loss": { "bootstrap_interval": [ 0.5115766962697499, 0.5165819343644471 ], "confidence_level": 0.95, "mean": 0.5140856217442001, "replicates": 8, "sample_sd": 0.003954051524564069, "standard_error": 0.0013979683230901298 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07239021582162622, 0.0793003924431713 ], "confidence_level": 0.95, "mean": 0.07589236968112664, "replicates": 8, "sample_sd": 0.005313849457926589, "standard_error": 0.0018787294929521752 }, "log_loss": { "bootstrap_interval": [ 0.33634452958201055, 0.398133582344796 ], "confidence_level": 0.95, "mean": 0.36486934566611184, "replicates": 8, "sample_sd": 0.04696590437497136, "standard_error": 0.016604954734050593 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19137060121273936, 0.1987320780838339 ], "confidence_level": 0.95, "mean": 0.1949771317143506, "replicates": 8, "sample_sd": 0.0056748999697914014, "standard_error": 0.002006380125597417 }, "log_loss": { "bootstrap_interval": [ 0.5636441956134586, 0.5808540152623622 ], "confidence_level": 0.95, "mean": 0.5721751720696512, "replicates": 8, "sample_sd": 0.013264221943656803, "standard_error": 0.004689610641761566 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06432240441986099, 0.0704799491602418 ], "confidence_level": 0.95, "mean": 0.06741256584075515, "replicates": 8, "sample_sd": 0.004760341260743628, "standard_error": 0.001683034793116969 }, "log_loss": { "bootstrap_interval": [ 0.2130511265154367, 0.23426049772181734 ], "confidence_level": 0.95, "mean": 0.22305796591158258, "replicates": 8, "sample_sd": 0.016308971459194386, "standard_error": 0.005766092156487106 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06422892079638608, 0.07040236306901043 ], "confidence_level": 0.95, "mean": 0.06737697353908852, "replicates": 8, "sample_sd": 0.004723264012811385, "standard_error": 0.0016699260063966573 }, "log_loss": { "bootstrap_interval": [ 0.21296340996502933, 0.23430344274021908 ], "confidence_level": 0.95, "mean": 0.22314265286725962, "replicates": 8, "sample_sd": 0.016641086752364632, "standard_error": 0.0058835126444553255 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3448275862068965 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3448275862068966, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.1-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05636527111958429, -0.0538489435487543 ], "confidence_level": 0.95, "mean": -0.05504556668258602, "replicates": 8, "sample_sd": 0.0019260181293892713, "standard_error": 0.0006809502399896915 }, "log_loss": { "bootstrap_interval": [ -0.2539306447653946, -0.24568482993898044 ], "confidence_level": 0.95, "mean": -0.24980982244063926, "replicates": 8, "sample_sd": 0.006422925574187418, "standard_error": 0.002270847114282211 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0011738111559437187, -7.569097545230313e-05 ], "confidence_level": 0.95, "mean": -0.0005061252178738114, "replicates": 8, "sample_sd": 0.0008933996552769375, "standard_error": 0.0003158644772780232 }, "log_loss": { "bootstrap_interval": [ -0.016949089296382523, -0.002361444897831137 ], "confidence_level": 0.95, "mean": -0.00835391735876386, "replicates": 8, "sample_sd": 0.011823648609893784, "standard_error": 0.004180291055211395 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.834032993410297e-05, 4.311930979074768e-05 ], "confidence_level": 0.95, "mean": -1.471379904097292e-05, "replicates": 8, "sample_sd": 0.00011345013576540359, "standard_error": 4.011068016312567e-05 }, "log_loss": { "bootstrap_interval": [ -0.0006531530754919434, 0.00012121641003996493 ], "confidence_level": 0.95, "mean": -0.00023806583785984268, "replicates": 8, "sample_sd": 0.0006026900381273503, "standard_error": 0.0002130831064567141 } } }, "discount": { "bootstrap_interval": [ 0.32127793207246935, 0.43620977535332567 ], "confidence_level": 0.95, "mean": 0.37225303361185536, "replicates": 8, "sample_sd": 0.08930676874291041, "standard_error": 0.03157471089198537 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.5629610748156659, "discount_min": 0.2953204482391476, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.057252905707189605, 0.05910215959362595 ], "confidence_level": 0.95, "mean": 0.05819237391798369, "replicates": 8, "sample_sd": 0.0014266464293977374, "standard_error": 0.0005043956822913576 }, "log_loss": { "bootstrap_interval": [ 0.2581476695170446, 0.2631870314114531 ], "confidence_level": 0.95, "mean": 0.2606932704413672, "replicates": 8, "sample_sd": 0.003854617024295076, "standard_error": 0.0013628129183780795 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0022288869041714367, 0.005602956028854597 ], "confidence_level": 0.95, "mean": 0.003652932453271484, "replicates": 8, "sample_sd": 0.0027346646362771615, "standard_error": 0.0009668499542913122 }, "log_loss": { "bootstrap_interval": [ 0.01015679377790705, 0.03205700471492528 ], "confidence_level": 0.95, "mean": 0.019237365359491783, "replicates": 8, "sample_sd": 0.017757901324775945, "standard_error": 0.006278366223195323 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10514688624325566, 0.11246687652425642 ], "confidence_level": 0.95, "mean": 0.10867614770258291, "replicates": 8, "sample_sd": 0.005710394374144836, "standard_error": 0.0020189292926036623 }, "log_loss": { "bootstrap_interval": [ 0.334764942223648, 0.3542808738534009 ], "confidence_level": 0.95, "mean": 0.34476458837454727, "replicates": 8, "sample_sd": 0.015212626970181132, "standard_error": 0.0053784758451382204 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0021683815769057368, 0.004497132984792587 ], "confidence_level": 0.95, "mean": 0.003146807235397673, "replicates": 8, "sample_sd": 0.0018594628917258011, "standard_error": 0.0006574194100520304 }, "log_loss": { "bootstrap_interval": [ 0.007726632716128779, 0.015301707145001427 ], "confidence_level": 0.95, "mean": 0.010883448000727925, "replicates": 8, "sample_sd": 0.006037423657503223, "standard_error": 0.0021345516045583085 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.002147571160513574, 0.00457171677424066 ], "confidence_level": 0.95, "mean": 0.0031615210344386455, "replicates": 8, "sample_sd": 0.0019520173766575599, "standard_error": 0.0006901423620142678 }, "log_loss": { "bootstrap_interval": [ 0.007870216427063237, 0.015918846840962106 ], "confidence_level": 0.95, "mean": 0.011121513838587768, "replicates": 8, "sample_sd": 0.00646523101132474, "standard_error": 0.0022858043450226418 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3448275862068965 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3448275862068966, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.25-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04971381607398666, -0.04552619144650494 ], "confidence_level": 0.95, "mean": -0.04768437953831868, "replicates": 8, "sample_sd": 0.003314149048949071, "standard_error": 0.0011717286331874176 }, "log_loss": { "bootstrap_interval": [ -0.14877204029999266, -0.13501438576694227 ], "confidence_level": 0.95, "mean": -0.14224921747903238, "replicates": 8, "sample_sd": 0.01061353109306758, "standard_error": 0.003752449904121178 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03282292155859504, -0.02858067092253788 ], "confidence_level": 0.95, "mean": -0.030852288921140676, "replicates": 8, "sample_sd": 0.003243261575608117, "standard_error": 0.001146666126637133 }, "log_loss": { "bootstrap_interval": [ -0.6781638270848205, -0.6054647367724278 ], "confidence_level": 0.95, "mean": -0.6411530101598059, "replicates": 8, "sample_sd": 0.055936902409224706, "standard_error": 0.019776681506066457 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.8578417249756904e-05, 0.00015795103430465266 ], "confidence_level": 0.95, "mean": 4.998870071957866e-05, "replicates": 8, "sample_sd": 0.0001472440253868281, "standard_error": 5.205862442011515e-05 }, "log_loss": { "bootstrap_interval": [ -0.00020963580077526522, 0.0001643264961183888 ], "confidence_level": 0.95, "mean": -9.953015648142549e-06, "replicates": 8, "sample_sd": 0.0002903981747597509, "standard_error": 0.00010267125930840798 } } }, "discount": { "bootstrap_interval": [ 0.16692433158877545, 0.18003731015490584 ], "confidence_level": 0.95, "mean": 0.17413597535660857, "replicates": 8, "sample_sd": 0.01010230104868668, "standard_error": 0.003571702788557161 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.1861457528103433, "discount_min": 0.1533804021459964, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1665707186214015, 0.17015200428931374 ], "confidence_level": 0.95, "mean": 0.16829946250867225, "replicates": 8, "sample_sd": 0.002843222641498293, "standard_error": 0.0010052310051132855 }, "log_loss": { "bootstrap_interval": [ 0.5173533715489593, 0.5253674303996094 ], "confidence_level": 0.95, "mean": 0.5211084752971392, "replicates": 8, "sample_sd": 0.0061722528074656035, "standard_error": 0.002182220907678317 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14658114221038573, 0.15664336949307175 ], "confidence_level": 0.95, "mean": 0.15146737189149426, "replicates": 8, "sample_sd": 0.0077178857342016325, "standard_error": 0.002728684669538445 }, "log_loss": { "bootstrap_interval": [ 0.974001317350861, 1.0657271949874316 ], "confidence_level": 0.95, "mean": 1.0200122679779127, "replicates": 8, "sample_sd": 0.07070990110133785, "standard_error": 0.024999725282893057 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19036843910668788, 0.19866410167051152 ], "confidence_level": 0.95, "mean": 0.1943748998290891, "replicates": 8, "sample_sd": 0.006438871471085747, "standard_error": 0.002276484840196666 }, "log_loss": { "bootstrap_interval": [ 0.5615814485761423, 0.580634797159652 ], "confidence_level": 0.95, "mean": 0.5710071403004052, "replicates": 8, "sample_sd": 0.014887286419211677, "standard_error": 0.0052634505902454855 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11715508604892072, 0.1243094418621446 ], "confidence_level": 0.95, "mean": 0.12061508297035359, "replicates": 8, "sample_sd": 0.005598056054828596, "standard_error": 0.0019792116989158556 }, "log_loss": { "bootstrap_interval": [ 0.3690607412767213, 0.3890596559966092 ], "confidence_level": 0.95, "mean": 0.3788592578181068, "replicates": 8, "sample_sd": 0.015629932133191797, "standard_error": 0.00552601550043272 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11708922145791403, 0.12421907609402265 ], "confidence_level": 0.95, "mean": 0.12056509426963401, "replicates": 8, "sample_sd": 0.005511231505385269, "standard_error": 0.001948514585073434 }, "log_loss": { "bootstrap_interval": [ 0.36914269460390914, 0.3891234197926432 ], "confidence_level": 0.95, "mean": 0.37886921083375497, "replicates": 8, "sample_sd": 0.015429847088522446, "standard_error": 0.005455274754482864 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.17391304347826086 ], "oracle_method": "oracle", "population_optimal_scalar": 0.17391304347826086, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.25-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.043737538864582905, -0.04194286375769218 ], "confidence_level": 0.95, "mean": -0.04284898442145207, "replicates": 8, "sample_sd": 0.0013771630200244047, "standard_error": 0.0004869006551293008 }, "log_loss": { "bootstrap_interval": [ -0.19403416081038954, -0.18458060854269395 ], "confidence_level": 0.95, "mean": -0.18962745046840218, "replicates": 8, "sample_sd": 0.00735151914976273, "standard_error": 0.002599154521409994 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.005463812948736616, -0.0031863895230787613 ], "confidence_level": 0.95, "mean": -0.004222971415694244, "replicates": 8, "sample_sd": 0.0017636963978446836, "standard_error": 0.0006235608414351313 }, "log_loss": { "bootstrap_interval": [ -0.1956877048876397, -0.14670031591109334 ], "confidence_level": 0.95, "mean": -0.16682110418537188, "replicates": 8, "sample_sd": 0.03946927071432158, "standard_error": 0.013954494485292198 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -1.1136701421413022e-05, 3.787994605681947e-05 ], "confidence_level": 0.95, "mean": 1.2351646717479771e-05, "replicates": 8, "sample_sd": 3.7258304037475644e-05, "standard_error": 1.3172799720204574e-05 }, "log_loss": { "bootstrap_interval": [ -0.00030640249444911956, 0.0003813804996075227 ], "confidence_level": 0.95, "mean": 7.06138358976642e-05, "replicates": 8, "sample_sd": 0.00053833466285562, "standard_error": 0.00019033004532649134 } } }, "discount": { "bootstrap_interval": [ 0.16165471077330934, 0.17823508339111704 ], "confidence_level": 0.95, "mean": 0.17003543296326334, "replicates": 8, "sample_sd": 0.01295978363075025, "standard_error": 0.004581975444006958 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.18945768828700194, "discount_min": 0.14859957804560375, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06391912731480374, 0.06665863963019752 ], "confidence_level": 0.95, "mean": 0.06525153872648695, "replicates": 8, "sample_sd": 0.002151716725159271, "standard_error": 0.0007607467437763156 }, "log_loss": { "bootstrap_interval": [ 0.2634649601735816, 0.2709270375306943 ], "confidence_level": 0.95, "mean": 0.26707104645559615, "replicates": 8, "sample_sd": 0.005769627746571754, "standard_error": 0.0020398714522614735 } }, "independent": { "brier": { "bootstrap_interval": [ 0.024304924925061453, 0.02901166323299675 ], "confidence_level": 0.95, "mean": 0.02662552572072913, "replicates": 8, "sample_sd": 0.003629562854962048, "standard_error": 0.0012832442537432348 }, "log_loss": { "bootstrap_interval": [ 0.2200455974069972, 0.28054406253820974 ], "confidence_level": 0.95, "mean": 0.24426470017256582, "replicates": 8, "sample_sd": 0.04804219692683615, "standard_error": 0.016985481615032678 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10452629915131816, 0.11229735658633169 ], "confidence_level": 0.95, "mean": 0.10829839718771324, "replicates": 8, "sample_sd": 0.006040941932025175, "standard_error": 0.0021357955024445823 }, "log_loss": { "bootstrap_interval": [ 0.33343561883830386, 0.3546254340530316 ], "confidence_level": 0.95, "mean": 0.3440690215177385, "replicates": 8, "sample_sd": 0.016362456544759204, "standard_error": 0.005785001989834719 } }, "learned": { "brier": { "bootstrap_interval": [ 0.02093816735370534, 0.02402798538137607 ], "confidence_level": 0.95, "mean": 0.022402554305034888, "replicates": 8, "sample_sd": 0.0024249752732187326, "standard_error": 0.0008573582299513333 }, "log_loss": { "bootstrap_interval": [ 0.07223330556458213, 0.08368356628616445 ], "confidence_level": 0.95, "mean": 0.07744359598719394, "replicates": 8, "sample_sd": 0.008911803579507166, "standard_error": 0.003150798371836032 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.020897311229767526, 0.023997116096810437 ], "confidence_level": 0.95, "mean": 0.022390202658317405, "replicates": 8, "sample_sd": 0.002415084191248006, "standard_error": 0.000853861204383947 }, "log_loss": { "bootstrap_interval": [ 0.07211112111098679, 0.08387032060352008 ], "confidence_level": 0.95, "mean": 0.07737298215129629, "replicates": 8, "sample_sd": 0.009324832270356805, "standard_error": 0.0032968260658982227 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.17391304347826086 ], "oracle_method": "oracle", "population_optimal_scalar": 0.17391304347826086, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.5-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.016473134764691633, -0.013260913011408578 ], "confidence_level": 0.95, "mean": -0.014939617782262021, "replicates": 8, "sample_sd": 0.0024958952626614487, "standard_error": 0.0008824322326796447 }, "log_loss": { "bootstrap_interval": [ -0.049592458975964954, -0.040177551241067894 ], "confidence_level": 0.95, "mean": -0.045028427719495806, "replicates": 8, "sample_sd": 0.007294686744789762, "standard_error": 0.002579061231936231 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.061873205344222554, -0.05306510108352913 ], "confidence_level": 0.95, "mean": -0.057194299306847926, "replicates": 8, "sample_sd": 0.006823177086125522, "standard_error": 0.002412357393418012 }, "log_loss": { "bootstrap_interval": [ -1.644963054945, -1.5025352513310157 ], "confidence_level": 0.95, "mean": -1.573239646271016, "replicates": 8, "sample_sd": 0.11087942301377844, "standard_error": 0.03920179595354723 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00010171660734207301, 0.00013686354427144983 ], "confidence_level": 0.95, "mean": 1.5277295092753357e-05, "replicates": 8, "sample_sd": 0.00018820706800424047, "standard_error": 6.654124702651807e-05 }, "log_loss": { "bootstrap_interval": [ -0.0003334385545957566, 0.00048588762502254496 ], "confidence_level": 0.95, "mean": 9.497526111557075e-05, "replicates": 8, "sample_sd": 0.0006398014435396739, "standard_error": 0.00022620396966992271 } } }, "discount": { "bootstrap_interval": [ 0.09060068771569992, 0.0987950625138 ], "confidence_level": 0.95, "mean": 0.0950218932082863, "replicates": 8, "sample_sd": 0.006371830717800159, "standard_error": 0.0022527823545646195 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.10141461353909237, "discount_min": 0.08295445757420883, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17448042922149254, 0.17991240290596724 ], "confidence_level": 0.95, "mean": 0.17693618530397043, "replicates": 8, "sample_sd": 0.0041494202671387, "standard_error": 0.001467041604443335 }, "log_loss": { "bootstrap_interval": [ 0.5298506237571085, 0.5415366599646007 ], "confidence_level": 0.95, "mean": 0.5352797186861773, "replicates": 8, "sample_sd": 0.009156727574675144, "standard_error": 0.0032373920807653215 } }, "independent": { "brier": { "bootstrap_interval": [ 0.21157903853351528, 0.2282131447925734 ], "confidence_level": 0.95, "mean": 0.21919086682855632, "replicates": 8, "sample_sd": 0.012830024133256821, "standard_error": 0.004536098533706477 }, "log_loss": { "bootstrap_interval": [ 1.9854286018459475, 2.145901062284182 ], "confidence_level": 0.95, "mean": 2.0634909372376975, "replicates": 8, "sample_sd": 0.12581019051889972, "standard_error": 0.044480619429142734 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1895243088019354, 0.19872782156167246 ], "confidence_level": 0.95, "mean": 0.19388169492360882, "replicates": 8, "sample_sd": 0.0071470337450813765, "standard_error": 0.002526858013258064 }, "log_loss": { "bootstrap_interval": [ 0.5600309255616245, 0.5809762376425139 ], "confidence_level": 0.95, "mean": 0.5701575733519997, "replicates": 8, "sample_sd": 0.016369939259517546, "standard_error": 0.0057876475290083735 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15819183490002994, 0.16648586038897076 ], "confidence_level": 0.95, "mean": 0.16199656752170838, "replicates": 8, "sample_sd": 0.00643531188017186, "standard_error": 0.0022752263347599363 }, "log_loss": { "bootstrap_interval": [ 0.4807537466031126, 0.5013657359621492 ], "confidence_level": 0.95, "mean": 0.4902512909666815, "replicates": 8, "sample_sd": 0.016020115706156107, "standard_error": 0.00566396622560805 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1583009710328925, 0.16642509678094697 ], "confidence_level": 0.95, "mean": 0.16198129022661564, "replicates": 8, "sample_sd": 0.006322459018507982, "standard_error": 0.0022353268228805187 }, "log_loss": { "bootstrap_interval": [ 0.4808903401940803, 0.5009657587672508 ], "confidence_level": 0.95, "mean": 0.49015631570556595, "replicates": 8, "sample_sd": 0.01564798687670144, "standard_error": 0.005532398816216846 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.09523809523809523 ], "oracle_method": "oracle", "population_optimal_scalar": 0.09523809523809523, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.5-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.01941277934027074, -0.017907294701059902 ], "confidence_level": 0.95, "mean": -0.018616320448929212, "replicates": 8, "sample_sd": 0.0012065704489317962, "standard_error": 0.000426587073209485 }, "log_loss": { "bootstrap_interval": [ -0.08854149888661285, -0.08132537401016139 ], "confidence_level": 0.95, "mean": -0.08502920061070465, "replicates": 8, "sample_sd": 0.005573900183506895, "standard_error": 0.0019706713087073334 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.018473760553311954, -0.014657099895603069 ], "confidence_level": 0.95, "mean": -0.016619127733184903, "replicates": 8, "sample_sd": 0.003012919587819975, "standard_error": 0.001065227935858641 }, "log_loss": { "bootstrap_interval": [ -0.9261689515606119, -0.8121807240921226 ], "confidence_level": 0.95, "mean": -0.8690309871912607, "replicates": 8, "sample_sd": 0.08548588878198898, "standard_error": 0.030223825826751708 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -5.5053785475954464e-06, 0.00020990027444411558 ], "confidence_level": 0.95, "mean": 8.876952969123515e-05, "replicates": 8, "sample_sd": 0.0001699846003503817, "standard_error": 6.009863180252004e-05 }, "log_loss": { "bootstrap_interval": [ 6.413091240518864e-05, 0.000990295523378537 ], "confidence_level": 0.95, "mean": 0.0004889351356682677, "replicates": 8, "sample_sd": 0.0007283202455413913, "standard_error": 0.00025750009224888455 } } }, "discount": { "bootstrap_interval": [ 0.08986703892998162, 0.09895202785119168 ], "confidence_level": 0.95, "mean": 0.09431839396377896, "replicates": 8, "sample_sd": 0.006998772305912395, "standard_error": 0.002474439678745632 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.10683872632042893, "discount_min": 0.08266545922566387, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07605705442883384, 0.08026536518417685 ], "confidence_level": 0.95, "mean": 0.07804330324189074, "replicates": 8, "sample_sd": 0.0032742638248523014, "standard_error": 0.0011576270769734322 }, "log_loss": { "bootstrap_interval": [ 0.27716420012825393, 0.288425778198336 ], "confidence_level": 0.95, "mean": 0.2824240915141896, "replicates": 8, "sample_sd": 0.008573868377864802, "standard_error": 0.003031320235494553 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07171471934883554, 0.08045626502450727 ], "confidence_level": 0.95, "mean": 0.07604611052614643, "replicates": 8, "sample_sd": 0.0067385006879650145, "standard_error": 0.0023824197657451385 }, "log_loss": { "bootstrap_interval": [ 1.0028551527777112, 1.1292885966449806 ], "confidence_level": 0.95, "mean": 1.0664258780947458, "replicates": 8, "sample_sd": 0.09714339496045349, "standard_error": 0.034345376662009874 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1036481228626028, 0.11195878294068182 ], "confidence_level": 0.95, "mean": 0.10774614617706736, "replicates": 8, "sample_sd": 0.006332313117656662, "standard_error": 0.002238810773045777 }, "log_loss": { "bootstrap_interval": [ 0.332202391494257, 0.3546786410639861 ], "confidence_level": 0.95, "mean": 0.3436452340089625, "replicates": 8, "sample_sd": 0.01747682052815941, "standard_error": 0.0061789891545208885 } }, "learned": { "brier": { "bootstrap_interval": [ 0.05693959972375564, 0.06204796342386906 ], "confidence_level": 0.95, "mean": 0.059426982792961526, "replicates": 8, "sample_sd": 0.003971633410423503, "standard_error": 0.0014041844584487565 }, "log_loss": { "bootstrap_interval": [ 0.189573182923448, 0.20542484697215257 ], "confidence_level": 0.95, "mean": 0.19739489090348497, "replicates": 8, "sample_sd": 0.01212271903299335, "standard_error": 0.004286028417324412 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.05688222553912978, 0.0619300449436049 ], "confidence_level": 0.95, "mean": 0.05933821326327029, "replicates": 8, "sample_sd": 0.00392631992556717, "standard_error": 0.001388163722238203 }, "log_loss": { "bootstrap_interval": [ 0.18943531085782914, 0.2048181605007988 ], "confidence_level": 0.95, "mean": 0.1969059557678167, "replicates": 8, "sample_sd": 0.011871066051783985, "standard_error": 0.004197055652564935 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.09523809523809523 ], "oracle_method": "oracle", "population_optimal_scalar": 0.09523809523809523, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.75-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.004042072235334277, -0.002278141615818536 ], "confidence_level": 0.95, "mean": -0.0032019079227608646, "replicates": 8, "sample_sd": 0.001367271488161122, "standard_error": 0.0004834034705008758 }, "log_loss": { "bootstrap_interval": [ -0.012382820225039358, -0.007275316207987464 ], "confidence_level": 0.95, "mean": -0.009898464717129574, "replicates": 8, "sample_sd": 0.003956989879750244, "standard_error": 0.0013990071885289694 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.08288867995784197, -0.07051500860230889 ], "confidence_level": 0.95, "mean": -0.07614817988517768, "replicates": 8, "sample_sd": 0.009643101547898726, "standard_error": 0.003409351248094841 }, "log_loss": { "bootstrap_interval": [ -2.561180541784677, -2.346826866091 ], "confidence_level": 0.95, "mean": -2.4482877710799906, "replicates": 8, "sample_sd": 0.1689959310879404, "standard_error": 0.05974908443260856 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.0001418989263025222, 0.00018398024351764034 ], "confidence_level": 0.95, "mean": 2.465347917017177e-05, "replicates": 8, "sample_sd": 0.0002538831043634174, "standard_error": 8.97612323620322e-05 }, "log_loss": { "bootstrap_interval": [ -0.0003945867664447188, 0.000736290662560618 ], "confidence_level": 0.95, "mean": 0.0001873232802507424, "replicates": 8, "sample_sd": 0.000870548052241584, "standard_error": 0.0003077852155443824 } } }, "discount": { "bootstrap_interval": [ 0.061922766884670105, 0.0682869030455536 ], "confidence_level": 0.95, "mean": 0.06523572583408556, "replicates": 8, "sample_sd": 0.004935922533092191, "standard_error": 0.0017451121472804845 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.07092555030429393, "discount_min": 0.056487751123044674, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.182822503708786, 0.18964889677847452 ], "confidence_level": 0.95, "mean": 0.18589205608275097, "replicates": 8, "sample_sd": 0.005317835225207858, "standard_error": 0.0018801386744885839 }, "log_loss": { "bootstrap_interval": [ 0.5453382037149032, 0.5606703732529409 ], "confidence_level": 0.95, "mean": 0.552225999813022, "replicates": 8, "sample_sd": 0.011952043466252185, "standard_error": 0.004225685492011644 } }, "independent": { "brier": { "bootstrap_interval": [ 0.24959577386122284, 0.26997367311628395 ], "confidence_level": 0.95, "mean": 0.2588383280451678, "replicates": 8, "sample_sd": 0.01602008064637926, "standard_error": 0.005663953830105073 }, "log_loss": { "bootstrap_interval": [ 2.8806672317930966, 3.120539932522637 ], "confidence_level": 0.95, "mean": 2.990615306175883, "replicates": 8, "sample_sd": 0.18384383491938205, "standard_error": 0.06499861117541762 } }, "individual": { "brier": { "bootstrap_interval": [ 0.18939529505873828, 0.1988353692064086 ], "confidence_level": 0.95, "mean": 0.19386376085344048, "replicates": 8, "sample_sd": 0.007383401697858005, "standard_error": 0.002610426704389832 }, "log_loss": { "bootstrap_interval": [ 0.5593596470565282, 0.5815686325410016 ], "confidence_level": 0.95, "mean": 0.5700788344611047, "replicates": 8, "sample_sd": 0.016928713781026974, "standard_error": 0.005985204155665166 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17885051253390616, 0.1873042001722178 ], "confidence_level": 0.95, "mean": 0.18269014815999013, "replicates": 8, "sample_sd": 0.00659227579859597, "standard_error": 0.0023307214603195863 }, "log_loss": { "bootstrap_interval": [ 0.5329779517381823, 0.5532823445919566 ], "confidence_level": 0.95, "mean": 0.5423275350958924, "replicates": 8, "sample_sd": 0.015687261249570948, "standard_error": 0.005546284403908285 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1788423363047319, 0.18714908275495246 ], "confidence_level": 0.95, "mean": 0.18266549468081994, "replicates": 8, "sample_sd": 0.006475130005776333, "standard_error": 0.002289304168074467 }, "log_loss": { "bootstrap_interval": [ 0.5333637376870992, 0.5531865613297031 ], "confidence_level": 0.95, "mean": 0.5421402118156416, "replicates": 8, "sample_sd": 0.015317919570068502, "standard_error": 0.005415702400832781 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.06557377049180328 ], "oracle_method": "oracle", "population_optimal_scalar": 0.06557377049180328, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho0.75-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.005381218320907758, -0.004054377078044206 ], "confidence_level": 0.95, "mean": -0.004671539044308741, "replicates": 8, "sample_sd": 0.0010128252592425913, "standard_error": 0.0003580878044837296 }, "log_loss": { "bootstrap_interval": [ -0.024853877096878526, -0.019736266131035164 ], "confidence_level": 0.95, "mean": -0.022270883077101407, "replicates": 8, "sample_sd": 0.003964575820769486, "standard_error": 0.0014016892236971628 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03186680268143535, -0.02616032683733605 ], "confidence_level": 0.95, "mean": -0.028850742677990843, "replicates": 8, "sample_sd": 0.004383938176671746, "standard_error": 0.0015499562065135902 }, "log_loss": { "bootstrap_interval": [ -1.908007447715919, -1.7241716994077814 ], "confidence_level": 0.95, "mean": -1.8160321242848976, "replicates": 8, "sample_sd": 0.14340990725221517, "standard_error": 0.05070305895368759 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.5408534163394734e-05, 0.00022354306624385964 ], "confidence_level": 0.95, "mean": 7.70039395219177e-05, "replicates": 8, "sample_sd": 0.0002093901276555663, "standard_error": 7.403058958938387e-05 }, "log_loss": { "bootstrap_interval": [ -4.904689525737816e-05, 0.0010359414334046024 ], "confidence_level": 0.95, "mean": 0.0004705212233773043, "replicates": 8, "sample_sd": 0.0008521599911474047, "standard_error": 0.00030128405419809905 } } }, "discount": { "bootstrap_interval": [ 0.062198859690099584, 0.0681188164133083 ], "confidence_level": 0.95, "mean": 0.06515573133284804, "replicates": 8, "sample_sd": 0.004618903895380696, "standard_error": 0.0016330291330363248 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.07240506950287776, "discount_min": 0.05743928194581832, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08958260089424233, 0.0954311552937058 ], "confidence_level": 0.95, "mean": 0.09230156039886331, "replicates": 8, "sample_sd": 0.004562913790797698, "standard_error": 0.001613233641721334 }, "log_loss": { "bootstrap_interval": [ 0.29932732818150665, 0.31449647851236956 ], "confidence_level": 0.95, "mean": 0.30637060465953125, "replicates": 8, "sample_sd": 0.011782877036558458, "standard_error": 0.004165876127218868 } }, "independent": { "brier": { "bootstrap_interval": [ 0.11181893571658522, 0.12203377151435038 ], "confidence_level": 0.95, "mean": 0.11648076403254541, "replicates": 8, "sample_sd": 0.007961068581485212, "standard_error": 0.002814662789729681 }, "log_loss": { "bootstrap_interval": [ 2.00171734003033, 2.202570307636604 ], "confidence_level": 0.95, "mean": 2.1001318458673275, "replicates": 8, "sample_sd": 0.1576027656929933, "standard_error": 0.05572099217763506 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10342893096572764, 0.1116979523217466 ], "confidence_level": 0.95, "mean": 0.10749807007090667, "replicates": 8, "sample_sd": 0.006495836244050999, "standard_error": 0.0022966249288229068 }, "log_loss": { "bootstrap_interval": [ 0.33172015712566855, 0.3551283538519174 ], "confidence_level": 0.95, "mean": 0.3436335678809739, "replicates": 8, "sample_sd": 0.01819648125975606, "standard_error": 0.00643342764625372 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08443912890653452, 0.09126491231000242 ], "confidence_level": 0.95, "mean": 0.08763002135455457, "replicates": 8, "sample_sd": 0.00528754449792581, "standard_error": 0.0018694292851544794 }, "log_loss": { "bootstrap_interval": [ 0.27494075865753087, 0.2942352275514323 ], "confidence_level": 0.95, "mean": 0.2840997215824298, "replicates": 8, "sample_sd": 0.014894778238811376, "standard_error": 0.005266099348466672 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08431280940204802, 0.09110916992412223 ], "confidence_level": 0.95, "mean": 0.08755301741503264, "replicates": 8, "sample_sd": 0.005233454714755071, "standard_error": 0.0018503056589180098 }, "log_loss": { "bootstrap_interval": [ 0.27471348483544356, 0.29349911664836337 ], "confidence_level": 0.95, "mean": 0.28362920035905254, "replicates": 8, "sample_sd": 0.014612634932300056, "standard_error": 0.0051663466258163974 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.06557377049180328 ], "oracle_method": "oracle", "population_optimal_scalar": 0.06557377049180328, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho1-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.00016657225458190222, 0.0002691713211540221 ], "confidence_level": 0.95, "mean": 5.417565099974722e-05, "replicates": 8, "sample_sd": 0.00033743856054865254, "standard_error": 0.0001193025471988898 }, "log_loss": { "bootstrap_interval": [ -0.00041108645686839095, 0.0010282947583384189 ], "confidence_level": 0.95, "mean": 0.00031282080976345605, "replicates": 8, "sample_sd": 0.0011149038923301054, "standard_error": 0.00039417805131894694 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.09725148292422725, -0.08301669989413665 ], "confidence_level": 0.95, "mean": -0.08974090567885885, "replicates": 8, "sample_sd": 0.011101415279678466, "standard_error": 0.003924943012514298 }, "log_loss": { "bootstrap_interval": [ -3.4005616800788245, -3.1082091320162615 ], "confidence_level": 0.95, "mean": -3.240817427742723, "replicates": 8, "sample_sd": 0.22904023650041697, "standard_error": 0.08097795219700771 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00016657225458190222, 0.0002691713211540221 ], "confidence_level": 0.95, "mean": 5.417565099974722e-05, "replicates": 8, "sample_sd": 0.00033743856054865254, "standard_error": 0.0001193025471988898 }, "log_loss": { "bootstrap_interval": [ -0.00041108645686839095, 0.0010282947583384189 ], "confidence_level": 0.95, "mean": 0.00031282080976345605, "replicates": 8, "sample_sd": 0.0011149038923301054, "standard_error": 0.00039417805131894694 } } }, "discount": { "bootstrap_interval": [ 0.04682293086688893, 0.05219879668851878 ], "confidence_level": 0.95, "mean": 0.04957637618343602, "replicates": 8, "sample_sd": 0.004153988128480037, "standard_error": 0.0014686565873083248 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.054533321517798466, "discount_min": 0.04273494987051203, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19111185909425413, 0.1996228712184227 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5634295288663593, 0.582880633874353 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2744278503283861, 0.2970285762225038 ], "confidence_level": 0.95, "mean": 0.2847660985581254, "replicates": 8, "sample_sd": 0.01745480248908296, "standard_error": 0.006171204602151195 }, "log_loss": { "bootstrap_interval": [ 3.6649473980798106, 3.9850735905417314 ], "confidence_level": 0.95, "mean": 3.8130255709862593, "replicates": 8, "sample_sd": 0.2440307174913563, "standard_error": 0.08627788757797833 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19119687266354665, 0.19953091451482902 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5634113622879122, 0.582743536600873 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19109383970946192, 0.19967892633857023 ], "confidence_level": 0.95, "mean": 0.19502519287926653, "replicates": 8, "sample_sd": 0.006700670866674529, "standard_error": 0.00236904490416235 }, "log_loss": { "bootstrap_interval": [ 0.5632987730119094, 0.5826278873184798 ], "confidence_level": 0.95, "mean": 0.5722081432435364, "replicates": 8, "sample_sd": 0.0154746166590676, "standard_error": 0.005471103187944508 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19118667955245405, 0.19960669334812753 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.56343065357251, 0.5824409087463771 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.05 ], "oracle_method": "oracle", "population_optimal_scalar": 0.05, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n20-rho1-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -8.763724911134006e-05, 0.00017543303982253504 ], "confidence_level": 0.95, "mean": 3.281577390299925e-05, "replicates": 8, "sample_sd": 0.00020814777778857522, "standard_error": 7.359135258160608e-05 }, "log_loss": { "bootstrap_interval": [ -0.00015462004548458946, 0.0011545309557987604 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.039917999351464145, -0.03278811003400582 ], "confidence_level": 0.95, "mean": -0.03653397336166019, "replicates": 8, "sample_sd": 0.005617441581747283, "standard_error": 0.0019860655176863946 }, "log_loss": { "bootstrap_interval": [ -2.989156286537412, -2.694133498702718 ], "confidence_level": 0.95, "mean": -2.83413829769731, "replicates": 8, "sample_sd": 0.22733045820872788, "standard_error": 0.08037345428481826 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -8.763724911134006e-05, 0.00017543303982253504 ], "confidence_level": 0.95, "mean": 3.281577390299925e-05, "replicates": 8, "sample_sd": 0.00020814777778857522, "standard_error": 7.359135258160608e-05 }, "log_loss": { "bootstrap_interval": [ -0.00015462004548458946, 0.0011545309557987604 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } } }, "discount": { "bootstrap_interval": [ 0.04728833666560386, 0.05181934212550593 ], "confidence_level": 0.95, "mean": 0.04959384119357672, "replicates": 8, "sample_sd": 0.0034948970894948307, "standard_error": 0.0012356327157654614 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.05397081015060036, "discount_min": 0.04394751545183566, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.10466872472665731, 0.11282178756054197 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.33495183872400125, 0.3566320079237723 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1387519379828248, 0.15178359989135348 ], "confidence_level": 0.95, "mean": 0.14493526749817381, "replicates": 8, "sample_sd": 0.010194040551109113, "standard_error": 0.0036041376006899517 }, "log_loss": { "bootstrap_interval": [ 3.02827460906171, 3.346670415663311 ], "confidence_level": 0.95, "mean": 3.1794479954899857, "replicates": 8, "sample_sd": 0.24338152790802464, "standard_error": 0.08604836439965359 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10462241925464807, 0.11274670980939391 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3347491182928781, 0.3568553562801791 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1045512616740912, 0.11286375430647794 ], "confidence_level": 0.95, "mean": 0.10840129413651362, "replicates": 8, "sample_sd": 0.006334716340172473, "standard_error": 0.002239660440514592 }, "log_loss": { "bootstrap_interval": [ 0.3348876748989158, 0.357233442716817 ], "confidence_level": 0.95, "mean": 0.3453096977926756, "replicates": 8, "sample_sd": 0.017009453256656443, "standard_error": 0.006013749871028687 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10473628070857156, 0.11281257501825454 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3348874515018528, 0.35664147600853585 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.05 ], "oracle_method": "oracle", "population_optimal_scalar": 0.05, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07466619805357305, -0.07018444582848948 ], "confidence_level": 0.95, "mean": -0.07256510931449121, "replicates": 8, "sample_sd": 0.0034673218929663742, "standard_error": 0.0012258834115365497 }, "log_loss": { "bootstrap_interval": [ -0.22270288263030874, -0.2078744809529037 ], "confidence_level": 0.95, "mean": -0.21625147868936637, "replicates": 8, "sample_sd": 0.01159196234481117, "standard_error": 0.004098377590637545 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -4.2322970314664574e-05, 0.0004188763470885249 ], "confidence_level": 0.95, "mean": 0.000139045750948516, "replicates": 8, "sample_sd": 0.0003821328785869947, "standard_error": 0.0001351043748815998 }, "log_loss": { "bootstrap_interval": [ -0.0003415704257923313, 0.0020990404332798737 ], "confidence_level": 0.95, "mean": 0.0006514577860421977, "replicates": 8, "sample_sd": 0.001956674241925455, "standard_error": 0.0006917888125192683 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -4.2322970314664574e-05, 0.0004188763470885249 ], "confidence_level": 0.95, "mean": 0.000139045750948516, "replicates": 8, "sample_sd": 0.0003821328785869947, "standard_error": 0.0001351043748815998 }, "log_loss": { "bootstrap_interval": [ -0.0003415704257923313, 0.0020990404332798737 ], "confidence_level": 0.95, "mean": 0.0006514577860421977, "replicates": 8, "sample_sd": 0.001956674241925455, "standard_error": 0.0006917888125192683 } } }, "discount": { "bootstrap_interval": [ 0.9261646754194821, 0.9854275070641654 ], "confidence_level": 0.95, "mean": 0.9592331850014406, "replicates": 8, "sample_sd": 0.046188778165298434, "standard_error": 0.016330199127701832 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8615011886365016, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1645526490249452, 0.1676333471987133 ], "confidence_level": 0.95, "mean": 0.16610141095895675, "replicates": 8, "sample_sd": 0.0023592541279959856, "standard_error": 0.0008341222962241581 }, "log_loss": { "bootstrap_interval": [ 0.515313117521666, 0.5216838012984092 ], "confidence_level": 0.95, "mean": 0.5184889057687017, "replicates": 8, "sample_sd": 0.004950845109706943, "standard_error": 0.0017503880748390179 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08983697199386236, 0.09680310013705493 ], "confidence_level": 0.95, "mean": 0.09339725589351701, "replicates": 8, "sample_sd": 0.005453279978879795, "standard_error": 0.0019280256263873676 }, "log_loss": { "bootstrap_interval": [ 0.29144960586420016, 0.31260633095636353 ], "confidence_level": 0.95, "mean": 0.3015859692932931, "replicates": 8, "sample_sd": 0.016032268400606533, "standard_error": 0.005668262851935842 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19325741680707073, 0.199075729998605 ], "confidence_level": 0.95, "mean": 0.19616138579072515, "replicates": 8, "sample_sd": 0.004520943891208201, "standard_error": 0.001598395041418608 }, "log_loss": { "bootstrap_interval": [ 0.5684037221748757, 0.5818372655617369 ], "confidence_level": 0.95, "mean": 0.5749023486997822, "replicates": 8, "sample_sd": 0.010327742370136919, "standard_error": 0.0036514083321357205 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0901779639294065, 0.09691671735714749 ], "confidence_level": 0.95, "mean": 0.09353630164446552, "replicates": 8, "sample_sd": 0.0051610424322973605, "standard_error": 0.0018247040509344883 }, "log_loss": { "bootstrap_interval": [ 0.2932712936673928, 0.3122718098920306 ], "confidence_level": 0.95, "mean": 0.3022374270793353, "replicates": 8, "sample_sd": 0.014573569386875682, "standard_error": 0.005152534869776235 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08985000199547305, 0.09677823544343532 ], "confidence_level": 0.95, "mean": 0.09339725589351701, "replicates": 8, "sample_sd": 0.005453279978879795, "standard_error": 0.0019280256263873676 }, "log_loss": { "bootstrap_interval": [ 0.2913609790674466, 0.31242041284172833 ], "confidence_level": 0.95, "mean": 0.3015859692932931, "replicates": 8, "sample_sd": 0.016032268400606533, "standard_error": 0.005668262851935842 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0545093600556067, -0.050876894402886855 ], "confidence_level": 0.95, "mean": -0.05273479180758827, "replicates": 8, "sample_sd": 0.0028214182668171205, "standard_error": 0.0009975219945149909 }, "log_loss": { "bootstrap_interval": [ -0.235985697119905, -0.2233653423711932 ], "confidence_level": 0.95, "mean": -0.22991938672735684, "replicates": 8, "sample_sd": 0.00996144138564072, "standard_error": 0.0035219013770894357 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.00010375806394009282, 9.728924889690436e-05 ], "confidence_level": 0.95, "mean": -4.056144887333579e-06, "replicates": 8, "sample_sd": 0.0001572347961672845, "standard_error": 5.5590895304185716e-05 }, "log_loss": { "bootstrap_interval": [ -0.0008198251603115259, 0.0012074591974749098 ], "confidence_level": 0.95, "mean": 0.00016106791527387056, "replicates": 8, "sample_sd": 0.0015343292292515998, "standard_error": 0.0005424673012882675 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00010375806394009282, 9.728924889690436e-05 ], "confidence_level": 0.95, "mean": -4.056144887333579e-06, "replicates": 8, "sample_sd": 0.0001572347961672845, "standard_error": 5.5590895304185716e-05 }, "log_loss": { "bootstrap_interval": [ -0.0008198251603115259, 0.0012074591974749098 ], "confidence_level": 0.95, "mean": 0.00016106791527387056, "replicates": 8, "sample_sd": 0.0015343292292515998, "standard_error": 0.0005424673012882675 } } }, "discount": { "bootstrap_interval": [ 0.833905103483589, 0.9566285811263464 ], "confidence_level": 0.95, "mean": 0.8942975105968889, "replicates": 8, "sample_sd": 0.09712652070265601, "standard_error": 0.034339410710951826 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7627277865375761, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0610978351475805, 0.06367320550334829 ], "confidence_level": 0.95, "mean": 0.06246629039060869, "replicates": 8, "sample_sd": 0.0019908534227052574, "standard_error": 0.0007038729777716678 }, "log_loss": { "bootstrap_interval": [ 0.26122234762638286, 0.2681284222100172 ], "confidence_level": 0.95, "mean": 0.26482298647879654, "replicates": 8, "sample_sd": 0.005279458371935005, "standard_error": 0.001866570407893666 } }, "independent": { "brier": { "bootstrap_interval": [ 0.00822722836421182, 0.011631622575623102 ], "confidence_level": 0.95, "mean": 0.009735554727907757, "replicates": 8, "sample_sd": 0.0026966212080208186, "standard_error": 0.0009533995712414902 }, "log_loss": { "bootstrap_interval": [ 0.028918216741910974, 0.041998249608259604 ], "confidence_level": 0.95, "mean": 0.03474253183616583, "replicates": 8, "sample_sd": 0.0102250104846692, "standard_error": 0.003615087125706569 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10600298938723025, 0.11222741408036613 ], "confidence_level": 0.95, "mean": 0.10914404534872604, "replicates": 8, "sample_sd": 0.004794727973542504, "standard_error": 0.0016951923320183688 }, "log_loss": { "bootstrap_interval": [ 0.3389327886667922, 0.35493106244526973 ], "confidence_level": 0.95, "mean": 0.3469855414655005, "replicates": 8, "sample_sd": 0.012538523242686587, "standard_error": 0.0044330374054844125 } }, "learned": { "brier": { "bootstrap_interval": [ 0.008319625743320942, 0.01158067876397942 ], "confidence_level": 0.95, "mean": 0.009731498583020423, "replicates": 8, "sample_sd": 0.002565451136514272, "standard_error": 0.0009070239477159884 }, "log_loss": { "bootstrap_interval": [ 0.02971643026567981, 0.041244346296029355 ], "confidence_level": 0.95, "mean": 0.0349035997514397, "replicates": 8, "sample_sd": 0.009017064398883926, "standard_error": 0.003188013691423312 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.008253917928443188, 0.011677955283659797 ], "confidence_level": 0.95, "mean": 0.009735554727907757, "replicates": 8, "sample_sd": 0.0026966212080208186, "standard_error": 0.0009533995712414902 }, "log_loss": { "bootstrap_interval": [ 0.02892388543858554, 0.042070838332017906 ], "confidence_level": 0.95, "mean": 0.03474253183616583, "replicates": 8, "sample_sd": 0.0102250104846692, "standard_error": 0.003615087125706569 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.1-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05209337848517327, -0.048836064038220155 ], "confidence_level": 0.95, "mean": -0.05064380279368541, "replicates": 8, "sample_sd": 0.002523541694993866, "standard_error": 0.0008922067225685784 }, "log_loss": { "bootstrap_interval": [ -0.15434570213267956, -0.14317127972070007 ], "confidence_level": 0.95, "mean": -0.14907709722309576, "replicates": 8, "sample_sd": 0.008613837668882048, "standard_error": 0.003045451513853309 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0021073956886872077, -0.0009404361438964174 ], "confidence_level": 0.95, "mean": -0.0015412128962901516, "replicates": 8, "sample_sd": 0.0009089526486095094, "standard_error": 0.00032136329080462855 }, "log_loss": { "bootstrap_interval": [ -0.016928167208928625, -0.008699563582092534 ], "confidence_level": 0.95, "mean": -0.01279068465080941, "replicates": 8, "sample_sd": 0.006302499654942378, "standard_error": 0.0022282701222178154 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.917091523399351e-06, 0.00025546732682116095 ], "confidence_level": 0.95, "mean": 0.00012285088781856593, "replicates": 8, "sample_sd": 0.0001921483105286709, "standard_error": 6.793468668418083e-05 }, "log_loss": { "bootstrap_interval": [ 8.80669190578453e-06, 0.0012179049422585814 ], "confidence_level": 0.95, "mean": 0.000546828168128545, "replicates": 8, "sample_sd": 0.0009404523350125668, "standard_error": 0.0003325001117350544 } } }, "discount": { "bootstrap_interval": [ 0.6774197455361987, 0.7108033464113475 ], "confidence_level": 0.95, "mean": 0.6930475615603275, "replicates": 8, "sample_sd": 0.025468983792528252, "standard_error": 0.0090046455748135 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.737576340173552, "discount_min": 0.6640768470951102, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16623776864461556, 0.16966280104234624 ], "confidence_level": 0.95, "mean": 0.16796073407949558, "replicates": 8, "sample_sd": 0.0026484160354793454, "standard_error": 0.0009363564690453185 }, "log_loss": { "bootstrap_interval": [ 0.5168848039122156, 0.5245371123612544 ], "confidence_level": 0.95, "mean": 0.5206180705198102, "replicates": 8, "sample_sd": 0.0058397913656355206, "standard_error": 0.0020646780376777627 } }, "independent": { "brier": { "bootstrap_interval": [ 0.11578899639680611, 0.12170392795408864 ], "confidence_level": 0.95, "mean": 0.1188581441821003, "replicates": 8, "sample_sd": 0.004570816115879033, "standard_error": 0.0016160275355474102 }, "log_loss": { "bootstrap_interval": [ 0.37329060145205695, 0.3955918675311314 ], "confidence_level": 0.95, "mean": 0.3843316579475239, "replicates": 8, "sample_sd": 0.017362383115980276, "standard_error": 0.006138529419434236 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19138871187141554, 0.19859035444585657 ], "confidence_level": 0.95, "mean": 0.1949771317143506, "replicates": 8, "sample_sd": 0.0056748999697914014, "standard_error": 0.002006380125597417 }, "log_loss": { "bootstrap_interval": [ 0.5636949672678646, 0.580870146010468 ], "confidence_level": 0.95, "mean": 0.5721751720696512, "replicates": 8, "sample_sd": 0.013264221943656803, "standard_error": 0.004689610641761566 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11469840195324008, 0.11983945107158105 ], "confidence_level": 0.95, "mean": 0.11731693128581015, "replicates": 8, "sample_sd": 0.0039795480591691725, "standard_error": 0.001406982709348143 }, "log_loss": { "bootstrap_interval": [ 0.36355275947432986, 0.3792799645764496 ], "confidence_level": 0.95, "mean": 0.3715409732967144, "replicates": 8, "sample_sd": 0.012185746105162331, "standard_error": 0.004308311852388922 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11440863718884654, 0.1198023396588907 ], "confidence_level": 0.95, "mean": 0.11719408039799159, "replicates": 8, "sample_sd": 0.0040821520139575134, "standard_error": 0.0014432586854518398 }, "log_loss": { "bootstrap_interval": [ 0.3630097860369223, 0.3791516200979097 ], "confidence_level": 0.95, "mean": 0.3709941451285859, "replicates": 8, "sample_sd": 0.012615948935330473, "standard_error": 0.004460411521637691 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.7142857142857143 ], "oracle_method": "oracle", "population_optimal_scalar": 0.7142857142857143, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.1-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.046330596157190096, -0.04262116556504134 ], "confidence_level": 0.95, "mean": -0.04446188368424052, "replicates": 8, "sample_sd": 0.002876568718847949, "standard_error": 0.001017020623823242 }, "log_loss": { "bootstrap_interval": [ -0.19800878092592072, -0.18586443459580437 ], "confidence_level": 0.95, "mean": -0.1918941289368883, "replicates": 8, "sample_sd": 0.009384291401030873, "standard_error": 0.0033178480431497685 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0007185552528677065, 0.00010345375968803935 ], "confidence_level": 0.95, "mean": -0.0003233403884812451, "replicates": 8, "sample_sd": 0.000635650302245517, "standard_error": 0.00022473631959054179 }, "log_loss": { "bootstrap_interval": [ -0.008331725355506267, -0.0014584250122407496 ], "confidence_level": 0.95, "mean": -0.005014289126378981, "replicates": 8, "sample_sd": 0.005281097459855899, "standard_error": 0.0018671499129855784 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.5456455814363414e-05, 0.0002344402757206085 ], "confidence_level": 0.95, "mean": 9.66492751925934e-05, "replicates": 8, "sample_sd": 0.00020904657831625798, "standard_error": 7.390912655563534e-05 }, "log_loss": { "bootstrap_interval": [ -5.802587625434225e-05, 0.002068260498223147 ], "confidence_level": 0.95, "mean": 0.0009177265215319225, "replicates": 8, "sample_sd": 0.0016874338317701503, "standard_error": 0.0005965979526241365 } } }, "discount": { "bootstrap_interval": [ 0.624615678778312, 0.72184897265874 ], "confidence_level": 0.95, "mean": 0.6695355407669134, "replicates": 8, "sample_sd": 0.075939104236548, "standard_error": 0.02684852778144758 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8083458087640236, "discount_min": 0.5956058734870788, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06430825101232959, 0.06703945140692893 ], "confidence_level": 0.95, "mean": 0.06569909441988682, "replicates": 8, "sample_sd": 0.002101771652991399, "standard_error": 0.0007430884941679386 }, "log_loss": { "bootstrap_interval": [ 0.2631242415533096, 0.2706019826847807 ], "confidence_level": 0.95, "mean": 0.2668920908242024, "replicates": 8, "sample_sd": 0.005791931431302334, "standard_error": 0.002047756995620693 } }, "independent": { "brier": { "bootstrap_interval": [ 0.018988135440259363, 0.024262405775786863 ], "confidence_level": 0.95, "mean": 0.021560551124127546, "replicates": 8, "sample_sd": 0.004046291099010628, "standard_error": 0.0014305799373825914 }, "log_loss": { "bootstrap_interval": [ 0.07044950599161998, 0.08974953471383887 ], "confidence_level": 0.95, "mean": 0.08001225101369308, "replicates": 8, "sample_sd": 0.015128289919587092, "standard_error": 0.005348658194948061 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10506705825771193, 0.11236766063446833 ], "confidence_level": 0.95, "mean": 0.10867614770258291, "replicates": 8, "sample_sd": 0.005710394374144836, "standard_error": 0.0020189292926036623 }, "log_loss": { "bootstrap_interval": [ 0.33467371748605484, 0.35438847390807565 ], "confidence_level": 0.95, "mean": 0.34476458837454727, "replicates": 8, "sample_sd": 0.015212626970181132, "standard_error": 0.0053784758451382204 } }, "learned": { "brier": { "bootstrap_interval": [ 0.019041907255392766, 0.023547792737882173 ], "confidence_level": 0.95, "mean": 0.0212372107356463, "replicates": 8, "sample_sd": 0.003495123015289773, "standard_error": 0.0012357125925962858 }, "log_loss": { "bootstrap_interval": [ 0.06857966117904195, 0.08165453415915024 ], "confidence_level": 0.95, "mean": 0.0749979618873141, "replicates": 8, "sample_sd": 0.010223709256335794, "standard_error": 0.003614627072017357 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.018862207176168426, 0.02350081097374671 ], "confidence_level": 0.95, "mean": 0.02114056146045371, "replicates": 8, "sample_sd": 0.0036009293325589722, "standard_error": 0.001273120774812999 }, "log_loss": { "bootstrap_interval": [ 0.0669670391751181, 0.08129753700366345 ], "confidence_level": 0.95, "mean": 0.07408023536578218, "replicates": 8, "sample_sd": 0.011015820958199004, "standard_error": 0.0038946808499397034 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.7142857142857143 ], "oracle_method": "oracle", "population_optimal_scalar": 0.7142857142857143, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.25-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.030150773798206618, -0.02707410922776655 ], "confidence_level": 0.95, "mean": -0.028754453366119714, "replicates": 8, "sample_sd": 0.002406444077120697, "standard_error": 0.000850806462739124 }, "log_loss": { "bootstrap_interval": [ -0.09038850051818612, -0.0792383433641283 ], "confidence_level": 0.95, "mean": -0.08515214709290131, "replicates": 8, "sample_sd": 0.008639870807544872, "standard_error": 0.0030546556182953354 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.009888928414471164, -0.007909339425281995 ], "confidence_level": 0.95, "mean": -0.008928850529865692, "replicates": 8, "sample_sd": 0.0015477800109719951, "standard_error": 0.0005472228707716433 }, "log_loss": { "bootstrap_interval": [ -0.07901030581720882, -0.05988208821124907 ], "confidence_level": 0.95, "mean": -0.06882405533790123, "replicates": 8, "sample_sd": 0.014828094616942506, "standard_error": 0.005242523127857893 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -1.7362368680154986e-05, 0.0001902924468930367 ], "confidence_level": 0.95, "mean": 7.072184447032975e-05, "replicates": 8, "sample_sd": 0.000162700399934964, "standard_error": 5.752327804788817e-05 }, "log_loss": { "bootstrap_interval": [ -0.0002386285552505099, 0.0008268609212904038 ], "confidence_level": 0.95, "mean": 0.00024947650355321843, "replicates": 8, "sample_sd": 0.0008261579785939068, "standard_error": 0.000292090954497561 } } }, "discount": { "bootstrap_interval": [ 0.4762079388126015, 0.5049969161349677 ], "confidence_level": 0.95, "mean": 0.49058787441940555, "replicates": 8, "sample_sd": 0.02237899301993486, "standard_error": 0.007912168860261175 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.5209480571403404, "discount_min": 0.45920617967533295, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16993050276681768, 0.17441397960237537 ], "confidence_level": 0.95, "mean": 0.17202359177653134, "replicates": 8, "sample_sd": 0.0034485475010933996, "standard_error": 0.0012192456616335328 }, "log_loss": { "bootstrap_interval": [ 0.5222329219062039, 0.5322452804017865 ], "confidence_level": 0.95, "mean": 0.5268723399551924, "replicates": 8, "sample_sd": 0.007682651539640312, "standard_error": 0.002716227500586467 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14819572093348332, 0.15634168543431154 ], "confidence_level": 0.95, "mean": 0.15219798894027733, "replicates": 8, "sample_sd": 0.006305984315884226, "standard_error": 0.0022295021359088735 }, "log_loss": { "bootstrap_interval": [ 0.4934684838223446, 0.5299557428693417 ], "confidence_level": 0.95, "mean": 0.5105442482001923, "replicates": 8, "sample_sd": 0.02813813485959107, "standard_error": 0.009948332984579214 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19036167167374046, 0.19865093242184131 ], "confidence_level": 0.95, "mean": 0.1943748998290891, "replicates": 8, "sample_sd": 0.006438871471085747, "standard_error": 0.002276484840196666 }, "log_loss": { "bootstrap_interval": [ 0.5615527078143077, 0.5807580285190557 ], "confidence_level": 0.95, "mean": 0.5710071403004052, "replicates": 8, "sample_sd": 0.014887286419211677, "standard_error": 0.0052634505902454855 } }, "learned": { "brier": { "bootstrap_interval": [ 0.13996200215434673, 0.14677150011323017 ], "confidence_level": 0.95, "mean": 0.14326913841041164, "replicates": 8, "sample_sd": 0.00525761066896996, "standard_error": 0.0018588460784336996 }, "log_loss": { "bootstrap_interval": [ 0.43255492003622603, 0.45171328603166383 ], "confidence_level": 0.95, "mean": 0.4417201928622911, "replicates": 8, "sample_sd": 0.015062539838850865, "standard_error": 0.005325412030971986 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.13989045538361639, 0.14666835586036678 ], "confidence_level": 0.95, "mean": 0.1431984165659413, "replicates": 8, "sample_sd": 0.005249640894513982, "standard_error": 0.0018560283376525247 }, "log_loss": { "bootstrap_interval": [ 0.43221182437836864, 0.4516889722813228 ], "confidence_level": 0.95, "mean": 0.44147071635873786, "replicates": 8, "sample_sd": 0.015070912186357234, "standard_error": 0.005328372102820088 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.25-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.033472139525623786, -0.029865279911007517 ], "confidence_level": 0.95, "mean": -0.03179828468475733, "replicates": 8, "sample_sd": 0.0028422882157091186, "standard_error": 0.001004900635707265 }, "log_loss": { "bootstrap_interval": [ -0.14223394124117303, -0.12945011618056754 ], "confidence_level": 0.95, "mean": -0.1365633132610298, "replicates": 8, "sample_sd": 0.009869665715373697, "standard_error": 0.0034894537776925592 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.003236679274964371, -0.0010953928141122808 ], "confidence_level": 0.95, "mean": -0.002131745370158705, "replicates": 8, "sample_sd": 0.001663998181778473, "standard_error": 0.0005883121991088218 }, "log_loss": { "bootstrap_interval": [ -0.04740367235458812, -0.0275778950987623 ], "confidence_level": 0.95, "mean": -0.036802005152275635, "replicates": 8, "sample_sd": 0.01513515508420415, "standard_error": 0.005351085397175403 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -4.073315867586679e-05, 0.0002220087639094321 ], "confidence_level": 0.95, "mean": 7.416472331904077e-05, "replicates": 8, "sample_sd": 0.0002020998710419684, "standard_error": 7.14530946453513e-05 }, "log_loss": { "bootstrap_interval": [ -0.00029183469070224316, 0.0015788241409779493 ], "confidence_level": 0.95, "mean": 0.0005414874566166631, "replicates": 8, "sample_sd": 0.0014120050392957696, "standard_error": 0.000499219169177808 } } }, "discount": { "bootstrap_interval": [ 0.45522682369541523, 0.4896012876024343 ], "confidence_level": 0.95, "mean": 0.47266448550034856, "replicates": 8, "sample_sd": 0.026526632246013416, "standard_error": 0.00937858077159891 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.5067746118646625, "discount_min": 0.4367697034315189, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06968667073243388, 0.07346010382433711 ], "confidence_level": 0.95, "mean": 0.07147836756881644, "replicates": 8, "sample_sd": 0.0029362906819900593, "standard_error": 0.0010381355263850217 }, "log_loss": { "bootstrap_interval": [ 0.26844290959344314, 0.27837034365864244 ], "confidence_level": 0.95, "mean": 0.27313415032036636, "replicates": 8, "sample_sd": 0.007741146913530554, "standard_error": 0.0027369087383593833 } }, "independent": { "brier": { "bootstrap_interval": [ 0.03813152241194507, 0.04570042496762748 ], "confidence_level": 0.95, "mean": 0.04181182825421781, "replicates": 8, "sample_sd": 0.00583554615406804, "standard_error": 0.0020631771287342943 }, "log_loss": { "bootstrap_interval": [ 0.15645259039693132, 0.1929790094215792 ], "confidence_level": 0.95, "mean": 0.1733728422116122, "replicates": 8, "sample_sd": 0.02850959145952414, "standard_error": 0.010079662724943799 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10451189055030233, 0.11217537055194239 ], "confidence_level": 0.95, "mean": 0.10829839718771324, "replicates": 8, "sample_sd": 0.006040941932025175, "standard_error": 0.0021357955024445823 }, "log_loss": { "bootstrap_interval": [ 0.33356057666193206, 0.35436103913214495 ], "confidence_level": 0.95, "mean": 0.3440690215177385, "replicates": 8, "sample_sd": 0.016362456544759204, "standard_error": 0.005785001989834719 } }, "learned": { "brier": { "bootstrap_interval": [ 0.03690132175615527, 0.04289789258907307 ], "confidence_level": 0.95, "mean": 0.03968008288405911, "replicates": 8, "sample_sd": 0.00453645867926951, "standard_error": 0.0016038803473420197 }, "log_loss": { "bootstrap_interval": [ 0.12840834475053203, 0.1459381893430388 ], "confidence_level": 0.95, "mean": 0.13657083705933656, "replicates": 8, "sample_sd": 0.013693473864031133, "standard_error": 0.004841374113628585 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.036727938031124785, 0.042819843565840995 ], "confidence_level": 0.95, "mean": 0.039605918160740064, "replicates": 8, "sample_sd": 0.0046413288555844185, "standard_error": 0.0016409575537502702 }, "log_loss": { "bootstrap_interval": [ 0.12750811094154788, 0.14611985277393869 ], "confidence_level": 0.95, "mean": 0.1360293496027199, "replicates": 8, "sample_sd": 0.014607156601458223, "standard_error": 0.0051644097433724764 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.5-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.011465289962627426, -0.008893519156779436 ], "confidence_level": 0.95, "mean": -0.010187048431027593, "replicates": 8, "sample_sd": 0.001953376560384791, "standard_error": 0.0006906229060294696 }, "log_loss": { "bootstrap_interval": [ -0.03497203115459833, -0.02675103889191941 ], "confidence_level": 0.95, "mean": -0.030922787696267974, "replicates": 8, "sample_sd": 0.006302749955764885, "standard_error": 0.0022283586169222812 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.02669271766273692, -0.02195938504457303 ], "confidence_level": 0.95, "mean": -0.024471540973610064, "replicates": 8, "sample_sd": 0.0036712262210847986, "standard_error": 0.001297974478099462 }, "log_loss": { "bootstrap_interval": [ -0.21818511749087005, -0.1834643299330717 ], "confidence_level": 0.95, "mean": -0.20003967753554863, "replicates": 8, "sample_sd": 0.02664716456311364, "standard_error": 0.009421195380985759 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00010151881145244179, 0.00018691659638407846 ], "confidence_level": 0.95, "mean": 4.1214156104307786e-05, "replicates": 8, "sample_sd": 0.00022042560132629275, "standard_error": 7.793221872247202e-05 }, "log_loss": { "bootstrap_interval": [ -0.00038102851281802643, 0.000727152224960767 ], "confidence_level": 0.95, "mean": 0.00015098863079307523, "replicates": 8, "sample_sd": 0.000866338384864856, "standard_error": 0.00030629687337007034 } } }, "discount": { "bootstrap_interval": [ 0.3156222806845096, 0.3441520026042972 ], "confidence_level": 0.95, "mean": 0.329839163549371, "replicates": 8, "sample_sd": 0.021677485106222975, "standard_error": 0.0076641483588403255 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3569038554238476, "discount_min": 0.29765224908533827, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17660643181680896, 0.18247677528429287 ], "confidence_level": 0.95, "mean": 0.179310340635496, "replicates": 8, "sample_sd": 0.004632050948780954, "standard_error": 0.001637677318342297 }, "log_loss": { "bootstrap_interval": [ 0.5332077230206776, 0.5465312191956161 ], "confidence_level": 0.95, "mean": 0.5394600022660361, "replicates": 8, "sample_sd": 0.010437569292430044, "standard_error": 0.003690238012890879 } }, "independent": { "brier": { "bootstrap_interval": [ 0.187320446098102, 0.1997773202988852 ], "confidence_level": 0.95, "mean": 0.19359483317807846, "replicates": 8, "sample_sd": 0.009665334507704779, "standard_error": 0.0034172117864171947 }, "log_loss": { "bootstrap_interval": [ 0.6836969797003575, 0.7366996484720996 ], "confidence_level": 0.95, "mean": 0.7085768921053167, "replicates": 8, "sample_sd": 0.041593870368742555, "standard_error": 0.014705653896766032 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1894657081289705, 0.19864145045404025 ], "confidence_level": 0.95, "mean": 0.19388169492360882, "replicates": 8, "sample_sd": 0.0071470337450813765, "standard_error": 0.002526858013258064 }, "log_loss": { "bootstrap_interval": [ 0.5596836548279362, 0.580604780764088 ], "confidence_level": 0.95, "mean": 0.5701575733519997, "replicates": 8, "sample_sd": 0.016369939259517546, "standard_error": 0.0057876475290083735 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16515933606985614, 0.17344059406794013 ], "confidence_level": 0.95, "mean": 0.1691232922044684, "replicates": 8, "sample_sd": 0.006392378161428008, "standard_error": 0.0022600469729272695 }, "log_loss": { "bootstrap_interval": [ 0.4985335386314808, 0.5197467951241792 ], "confidence_level": 0.95, "mean": 0.5085372145697681, "replicates": 8, "sample_sd": 0.016258155708758618, "standard_error": 0.0057481260756249986 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.16518105618014642, 0.17331156400068956 ], "confidence_level": 0.95, "mean": 0.1690820780483641, "replicates": 8, "sample_sd": 0.00630109235117908, "standard_error": 0.002227772565200707 }, "log_loss": { "bootstrap_interval": [ 0.498607971231089, 0.5189876631975594 ], "confidence_level": 0.95, "mean": 0.508386225938975, "replicates": 8, "sample_sd": 0.015981911159090312, "standard_error": 0.005650458878456857 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.5-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.015104135146252978, -0.013040668753324805 ], "confidence_level": 0.95, "mean": -0.014131191119748745, "replicates": 8, "sample_sd": 0.0015838276685195395, "standard_error": 0.0005599676423205228 }, "log_loss": { "bootstrap_interval": [ -0.06598562735099674, -0.05740510071778041 ], "confidence_level": 0.95, "mean": -0.06231833895602404, "replicates": 8, "sample_sd": 0.006823808820582368, "standard_error": 0.0024125807452771847 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.009828851311219236, -0.007150274656711881 ], "confidence_level": 0.95, "mean": -0.008593145522429489, "replicates": 8, "sample_sd": 0.002090233512655362, "standard_error": 0.0007390091455309918 }, "log_loss": { "bootstrap_interval": [ -0.16454264257337783, -0.1288631723110296 ], "confidence_level": 0.95, "mean": -0.14498522946128237, "replicates": 8, "sample_sd": 0.028406334051422146, "standard_error": 0.010043155718205465 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.706017744368813e-05, 0.00017908903154455202 ], "confidence_level": 0.95, "mean": 6.624953790513276e-05, "replicates": 8, "sample_sd": 0.00015939276030672153, "standard_error": 5.635385084246238e-05 }, "log_loss": { "bootstrap_interval": [ -0.000162793696105719, 0.000885108483086448 ], "confidence_level": 0.95, "mean": 0.0003382084655400193, "replicates": 8, "sample_sd": 0.0008186066333962966, "standard_error": 0.00028942115079940567 } } }, "discount": { "bootstrap_interval": [ 0.311432731528145, 0.3355878968815439 ], "confidence_level": 0.95, "mean": 0.32365549328299814, "replicates": 8, "sample_sd": 0.01878428825546022, "standard_error": 0.006641248802599371 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3524635343959731, "discount_min": 0.29230147441890353, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07966220825347668, 0.08501160917277052 ], "confidence_level": 0.95, "mean": 0.08216295494780936, "replicates": 8, "sample_sd": 0.004165330876528397, "standard_error": 0.0014726668543394675 }, "log_loss": { "bootstrap_interval": [ 0.28150638151953633, 0.29558765844680834 ], "confidence_level": 0.95, "mean": 0.2879940553301009, "replicates": 8, "sample_sd": 0.010965750724646704, "standard_error": 0.0038769783490994903 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0725504645874922, 0.08116411400051074 ], "confidence_level": 0.95, "mean": 0.0766249093504901, "replicates": 8, "sample_sd": 0.006574499862718989, "standard_error": 0.0023244367179193113 }, "log_loss": { "bootstrap_interval": [ 0.34625218961991, 0.40019837417175697 ], "confidence_level": 0.95, "mean": 0.37066094583535925, "replicates": 8, "sample_sd": 0.04279890889804854, "standard_error": 0.015131699354597695 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10359840747215403, 0.11195866017462927 ], "confidence_level": 0.95, "mean": 0.10774614617706736, "replicates": 8, "sample_sd": 0.006332313117656662, "standard_error": 0.002238810773045777 }, "log_loss": { "bootstrap_interval": [ 0.3322569116890844, 0.35470399839382305 ], "confidence_level": 0.95, "mean": 0.3436452340089625, "replicates": 8, "sample_sd": 0.01747682052815941, "standard_error": 0.0061789891545208885 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06503015783292942, 0.07145520366666783 ], "confidence_level": 0.95, "mean": 0.06803176382806062, "replicates": 8, "sample_sd": 0.004967014339037976, "standard_error": 0.001756104760692285 }, "log_loss": { "bootstrap_interval": [ 0.21642985138639187, 0.23630740052730453 ], "confidence_level": 0.95, "mean": 0.22567571637407685, "replicates": 8, "sample_sd": 0.01541049731002648, "standard_error": 0.005448433574688386 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.0649984676644565, 0.07139202109050873 ], "confidence_level": 0.95, "mean": 0.06796551429015549, "replicates": 8, "sample_sd": 0.004983236088043322, "standard_error": 0.001761840015054478 }, "log_loss": { "bootstrap_interval": [ 0.2161891196400899, 0.23627439785914564 ], "confidence_level": 0.95, "mean": 0.22533750790853685, "replicates": 8, "sample_sd": 0.015743469936408013, "standard_error": 0.005566157175720325 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.75-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.003185719896182944, -0.0016536788658622834 ], "confidence_level": 0.95, "mean": -0.0023874232564467003, "replicates": 8, "sample_sd": 0.0011647912274787406, "standard_error": 0.0004118158878084099 }, "log_loss": { "bootstrap_interval": [ -0.0096704751272009, -0.00515412842735008 ], "confidence_level": 0.95, "mean": -0.007416083397181111, "replicates": 8, "sample_sd": 0.0034795850884935986, "standard_error": 0.0012302191058947082 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04161764810218839, -0.03389583831919938 ], "confidence_level": 0.95, "mean": -0.037887455772011844, "replicates": 8, "sample_sd": 0.00598940373760954, "standard_error": 0.0021175739990638796 }, "log_loss": { "bootstrap_interval": [ -0.3694885118236466, -0.3202769839639706 ], "confidence_level": 0.95, "mean": -0.34347541955098637, "replicates": 8, "sample_sd": 0.037929685121108433, "standard_error": 0.013410168778703133 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00017231270471919358, 0.0002158299585296626 ], "confidence_level": 0.95, "mean": 2.8339291064636424e-05, "replicates": 8, "sample_sd": 0.0003031872285089938, "standard_error": 0.00010719287262393242 }, "log_loss": { "bootstrap_interval": [ -0.0004769782280408946, 0.0007931137511646242 ], "confidence_level": 0.95, "mean": 0.000168867396752731, "replicates": 8, "sample_sd": 0.0009967566360674217, "standard_error": 0.00035240668827798277 } } }, "discount": { "bootstrap_interval": [ 0.23551877914906214, 0.2600667038401901 ], "confidence_level": 0.95, "mean": 0.24803865293995506, "replicates": 8, "sample_sd": 0.019065775001944896, "standard_error": 0.006740769396226098 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.2698137394565766, "discount_min": 0.21856221106562684, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1835989938034914, 0.19087318125532757 ], "confidence_level": 0.95, "mean": 0.1869520454695762, "replicates": 8, "sample_sd": 0.005663334413919507, "standard_error": 0.0020022910841048123 }, "log_loss": { "bootstrap_interval": [ 0.546650450721479, 0.563354722449112 ], "confidence_level": 0.95, "mean": 0.5543454121759126, "replicates": 8, "sample_sd": 0.012914459060588024, "standard_error": 0.004565950788548921 } }, "independent": { "brier": { "bootstrap_interval": [ 0.214680518198221, 0.23062908220230424 ], "confidence_level": 0.95, "mean": 0.22245207798514136, "replicates": 8, "sample_sd": 0.012243795596432388, "standard_error": 0.004328835446849665 }, "log_loss": { "bootstrap_interval": [ 0.8585032736338172, 0.926230202502287 ], "confidence_level": 0.95, "mean": 0.8904047483297178, "replicates": 8, "sample_sd": 0.05322860949841329, "standard_error": 0.018819155364729355 } }, "individual": { "brier": { "bootstrap_interval": [ 0.18926217325180006, 0.19893203575224344 ], "confidence_level": 0.95, "mean": 0.19386376085344048, "replicates": 8, "sample_sd": 0.007383401697858005, "standard_error": 0.002610426704389832 }, "log_loss": { "bootstrap_interval": [ 0.5596267698496797, 0.581348742347864 ], "confidence_level": 0.95, "mean": 0.5700788344611047, "replicates": 8, "sample_sd": 0.016928713781026974, "standard_error": 0.005985204155665166 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18047263144368736, 0.189126910952749 ], "confidence_level": 0.95, "mean": 0.1845646222131295, "replicates": 8, "sample_sd": 0.006726693336664321, "standard_error": 0.0023782452366588524 }, "log_loss": { "bootstrap_interval": [ 0.5373242555505492, 0.5579680725921484 ], "confidence_level": 0.95, "mean": 0.5469293287787315, "replicates": 8, "sample_sd": 0.016129092181147983, "standard_error": 0.005702495227836331 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1805928447016949, 0.18907816377568026 ], "confidence_level": 0.95, "mean": 0.18453628292206486, "replicates": 8, "sample_sd": 0.006597546789928887, "standard_error": 0.002332585037177127 }, "log_loss": { "bootstrap_interval": [ 0.5374372474209973, 0.5573697554973984 ], "confidence_level": 0.95, "mean": 0.5467604613819788, "replicates": 8, "sample_sd": 0.015755376592743037, "standard_error": 0.005570366814438201 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25 ], "oracle_method": "oracle", "population_optimal_scalar": 0.25, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho0.75-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.004129068402542187, -0.0031365947673241776 ], "confidence_level": 0.95, "mean": -0.0036217325323750233, "replicates": 8, "sample_sd": 0.0007593088318477152, "standard_error": 0.00026845621200717763 }, "log_loss": { "bootstrap_interval": [ -0.018876980383895063, -0.01440314339404336 ], "confidence_level": 0.95, "mean": -0.016765399510463228, "replicates": 8, "sample_sd": 0.0035047629566356503, "standard_error": 0.001239120826544241 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.017411746380206438, -0.014347540314552798 ], "confidence_level": 0.95, "mean": -0.015886710183555357, "replicates": 8, "sample_sd": 0.00238024734432703, "standard_error": 0.000841544519037457 }, "log_loss": { "bootstrap_interval": [ -0.32661034473545497, -0.2783402882996798 ], "confidence_level": 0.95, "mean": -0.30111726292692287, "replicates": 8, "sample_sd": 0.03748969293105514, "standard_error": 0.013254608048075232 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.2213485738676212e-05, 0.0001608760911136313 ], "confidence_level": 0.95, "mean": 6.845749723089628e-05, "replicates": 8, "sample_sd": 0.000140615127429618, "standard_error": 4.97149550714467e-05 }, "log_loss": { "bootstrap_interval": [ -4.303600000125224e-05, 0.000827141229166829 ], "confidence_level": 0.95, "mean": 0.0003856282539898212, "replicates": 8, "sample_sd": 0.0006669939564889224, "standard_error": 0.00023581797482188102 } } }, "discount": { "bootstrap_interval": [ 0.23511061173373896, 0.2573582751730489 ], "confidence_level": 0.95, "mean": 0.24631166827243706, "replicates": 8, "sample_sd": 0.01706801941813245, "standard_error": 0.006034456135992562 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.27241611051713077, "discount_min": 0.22006273851624403, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09113681224823773, 0.09793722809974094 ], "confidence_level": 0.95, "mean": 0.09432249603088858, "replicates": 8, "sample_sd": 0.0052694636582076735, "standard_error": 0.0018630367429673586 }, "log_loss": { "bootstrap_interval": [ 0.30187960403567954, 0.3197407070488182 ], "confidence_level": 0.95, "mean": 0.3103003590894736, "replicates": 8, "sample_sd": 0.014074503389931211, "standard_error": 0.004976088394426705 } }, "independent": { "brier": { "bootstrap_interval": [ 0.10199747038086539, 0.1117787564202051 ], "confidence_level": 0.95, "mean": 0.1065874736820689, "replicates": 8, "sample_sd": 0.007610092388406047, "standard_error": 0.0026905739666490225 }, "log_loss": { "bootstrap_interval": [ 0.5629468730091373, 0.6316998646188778 ], "confidence_level": 0.95, "mean": 0.5946522225059332, "replicates": 8, "sample_sd": 0.053035126071270144, "standard_error": 0.01875074864303929 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10336272353945933, 0.11179830287236052 ], "confidence_level": 0.95, "mean": 0.10749807007090667, "replicates": 8, "sample_sd": 0.006495836244050999, "standard_error": 0.0022966249288229068 }, "log_loss": { "bootstrap_interval": [ 0.3319565482991793, 0.3553767404017701 ], "confidence_level": 0.95, "mean": 0.3436335678809739, "replicates": 8, "sample_sd": 0.01819648125975606, "standard_error": 0.00643342764625372 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08715080954708525, 0.09460094586751495 ], "confidence_level": 0.95, "mean": 0.09070076349851355, "replicates": 8, "sample_sd": 0.00576847628346596, "standard_error": 0.0020394643485762763 }, "log_loss": { "bootstrap_interval": [ 0.28330216761105703, 0.30483517586954345 ], "confidence_level": 0.95, "mean": 0.2935349595790104, "replicates": 8, "sample_sd": 0.016578268626342617, "standard_error": 0.005861303083009527 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08718751556100462, 0.09454353000149857 ], "confidence_level": 0.95, "mean": 0.09063230600128265, "replicates": 8, "sample_sd": 0.005732019657080719, "standard_error": 0.0020265749847081824 }, "log_loss": { "bootstrap_interval": [ 0.28328478008293045, 0.30430368851610073 ], "confidence_level": 0.95, "mean": 0.2931493313250206, "replicates": 8, "sample_sd": 0.016550880559210456, "standard_error": 0.005851619939013155 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25 ], "oracle_method": "oracle", "population_optimal_scalar": 0.25, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho1-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.00017114725154791368, 0.0002755190172585057 ], "confidence_level": 0.95, "mean": 5.417565099974375e-05, "replicates": 8, "sample_sd": 0.0003374385605486479, "standard_error": 0.00011930254719888816 }, "log_loss": { "bootstrap_interval": [ -0.00041262252157208844, 0.0010264571396639429 ], "confidence_level": 0.95, "mean": 0.00031282080976345605, "replicates": 8, "sample_sd": 0.0011149038923301054, "standard_error": 0.00039417805131894694 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.05408820426173593, -0.04376371056086232 ], "confidence_level": 0.95, "mean": -0.04868630350938821, "replicates": 8, "sample_sd": 0.008032247407733565, "standard_error": 0.0028398283050882355 }, "log_loss": { "bootstrap_interval": [ -0.5190349589168262, -0.45953643448584036 ], "confidence_level": 0.95, "mean": -0.48681897552941045, "replicates": 8, "sample_sd": 0.04706315647053938, "standard_error": 0.016639338542180967 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00017114725154791368, 0.00027551901725851943 ], "confidence_level": 0.95, "mean": 5.417565099974722e-05, "replicates": 8, "sample_sd": 0.00033743856054865254, "standard_error": 0.0001193025471988898 }, "log_loss": { "bootstrap_interval": [ -0.00041262252157208844, 0.0010264571396639429 ], "confidence_level": 0.95, "mean": 0.00031282080976345605, "replicates": 8, "sample_sd": 0.0011149038923301054, "standard_error": 0.00039417805131894694 } } }, "discount": { "bootstrap_interval": [ 0.18739792368492952, 0.20868561484860787 ], "confidence_level": 0.95, "mean": 0.1983055047337441, "replicates": 8, "sample_sd": 0.01661595251392015, "standard_error": 0.005874626349233299 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.21813328607119387, "discount_min": 0.17093979948204813, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19115494383422527, 0.19966054796962077 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5630342834390717, 0.5829336810904414 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "independent": { "brier": { "bootstrap_interval": [ 0.23537851692072873, 0.2535633718596124 ], "confidence_level": 0.95, "mean": 0.24371149638865475, "replicates": 8, "sample_sd": 0.014256272500120801, "standard_error": 0.005040353479639357 }, "log_loss": { "bootstrap_interval": [ 1.0226414808926472, 1.1034183143382594 ], "confidence_level": 0.95, "mean": 1.0590271187729468, "replicates": 8, "sample_sd": 0.0621713724359339, "standard_error": 0.02198089952256163 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19115882115959867, 0.19971561617704298 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.563028053659838, 0.5823349510593254 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19116218905633584, 0.1998027704051249 ], "confidence_level": 0.95, "mean": 0.19502519287926653, "replicates": 8, "sample_sd": 0.006700670866674529, "standard_error": 0.00236904490416235 }, "log_loss": { "bootstrap_interval": [ 0.563118064152219, 0.5831887236662013 ], "confidence_level": 0.95, "mean": 0.5722081432435364, "replicates": 8, "sample_sd": 0.0154746166590676, "standard_error": 0.005471103187944508 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19118666747670446, 0.1997262923181195 ], "confidence_level": 0.95, "mean": 0.1949710172282668, "replicates": 8, "sample_sd": 0.006609952900726725, "standard_error": 0.0023369712597137784 }, "log_loss": { "bootstrap_interval": [ 0.5631402219726195, 0.5828679842364878 ], "confidence_level": 0.95, "mean": 0.571895322433773, "replicates": 8, "sample_sd": 0.015223625524348782, "standard_error": 0.005382364421255816 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2 ], "oracle_method": "oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 }, "exchangeable-n5-rho1-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -8.704920854229851e-05, 0.0001818807646031366 ], "confidence_level": 0.95, "mean": 3.281577390299925e-05, "replicates": 8, "sample_sd": 0.00020814777778857522, "standard_error": 7.359135258160608e-05 }, "log_loss": { "bootstrap_interval": [ -0.00016651733860376992, 0.0011545309557987604 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.025842425804085897, -0.021180627742705876 ], "confidence_level": 0.95, "mean": -0.023513479298909158, "replicates": 8, "sample_sd": 0.0035843133544962505, "standard_error": 0.0012672461394309 }, "log_loss": { "bootstrap_interval": [ -0.514964291477486, -0.4571412802302287 ], "confidence_level": 0.95, "mean": -0.4848315490394327, "replicates": 8, "sample_sd": 0.044895678906182075, "standard_error": 0.015873019500267593 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -8.704920854229851e-05, 0.0001818807646031366 ], "confidence_level": 0.95, "mean": 3.281577390299925e-05, "replicates": 8, "sample_sd": 0.00020814777778857522, "standard_error": 7.359135258160608e-05 }, "log_loss": { "bootstrap_interval": [ -0.00016651733860376992, 0.0011545309557987604 ], "confidence_level": 0.95, "mean": 0.0004647113660391361, "replicates": 8, "sample_sd": 0.0010251655275458752, "standard_error": 0.00036245074818318636 } } }, "discount": { "bootstrap_interval": [ 0.1895526815593222, 0.20713222542022036 ], "confidence_level": 0.95, "mean": 0.19837536477430687, "replicates": 8, "sample_sd": 0.013979588357979323, "standard_error": 0.004942530863061846 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.21588324060240144, "discount_min": 0.17579006180734263, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1048161317012992, 0.11274736068877034 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3349936296924866, 0.3565360012286666 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "independent": { "brier": { "bootstrap_interval": [ 0.126358536323671, 0.13839872348329774 ], "confidence_level": 0.95, "mean": 0.13191477343542277, "replicates": 8, "sample_sd": 0.009309356337035436, "standard_error": 0.0032913544971998575 }, "log_loss": { "bootstrap_interval": [ 0.7929409636912652, 0.8714880287885312 ], "confidence_level": 0.95, "mean": 0.8301412468321083, "replicates": 8, "sample_sd": 0.0609507137262275, "standard_error": 0.021549331496987723 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1048527282742038, 0.11283394016204973 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.3349927649241213, 0.3565360012286666 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10457721349412409, 0.1128934641287982 ], "confidence_level": 0.95, "mean": 0.10840129413651362, "replicates": 8, "sample_sd": 0.006334716340172473, "standard_error": 0.002239660440514592 }, "log_loss": { "bootstrap_interval": [ 0.3352767879883409, 0.3572675622891107 ], "confidence_level": 0.95, "mean": 0.3453096977926756, "replicates": 8, "sample_sd": 0.017009453256656443, "standard_error": 0.006013749871028687 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10473578383798941, 0.11281279762763687 ], "confidence_level": 0.95, "mean": 0.10836847836261063, "replicates": 8, "sample_sd": 0.006270878159161954, "standard_error": 0.002217090235169016 }, "log_loss": { "bootstrap_interval": [ 0.33507492919196863, 0.35680872030062993 ], "confidence_level": 0.95, "mean": 0.34484498642663647, "replicates": 8, "sample_sd": 0.016842972410414735, "standard_error": 0.005954890003371094 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2 ], "oracle_method": "oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 1000 } }, "config": { "conditions": [ { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a1", "overlap": 1, "signal_strength": 1.0 } ], "phase": "exploratory-stress-exchangeable-m1000-v1", "replicates": 8, "schema_version": 1, "seed": 84173002, "test_count": 1000, "train_count": 1000 }, "config_sha256": "d550fc1d7de1ce96d9fd7f79c5f1fbebf2051f8dd5fed69cc26df3040c4850ea", "expected_runs": 384, "input_path": "results/stress-v1/stress-exchangeable-m1000-v1.json", "input_sha256": "f48e3530d32693017661dd8b66fdf03b79cb466fd2ca4f708c3bf441501ffa82", "ledger_receipt": { "bytes": 169424930, "count": 768000, "payload_sha256": "f9f704c08d4fb00eb0033b0bea313d2217aaa188b8886897350f0b94a874b2a4", "schema_version": 1, "sha256": "b33ff21bb83fb54d0501f84a3865e4d0d45e9f1d658c56a4060aa0d01b8afcaa" }, "manifest_sha256": null, "observed_runs": 384, "phase": "exploratory-stress-exchangeable-m1000-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" }, "exploratory-stress-exchangeable-m200-v1": { "analysis_seed": 159707918731458716086366102997467619628080349980308361867894291677213844435, "cells": { "exchangeable-n10-rho0-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12337427577808159, -0.11860717803011538 ], "confidence_level": 0.95, "mean": -0.12106673606296682, "replicates": 8, "sample_sd": 0.0037329582302798076, "standard_error": 0.0013198000392584926 }, "log_loss": { "bootstrap_interval": [ -0.37796998463991227, -0.36494128998629516 ], "confidence_level": 0.95, "mean": -0.37155000804520977, "replicates": 8, "sample_sd": 0.010088369916507167, "standard_error": 0.003566777389540291 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.00017519087226506528, 0.00010494189432102945 ], "confidence_level": 0.95, "mean": -2.3416325981345278e-05, "replicates": 8, "sample_sd": 0.00020430360985312392, "standard_error": 7.223223397401733e-05 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0006372243902011687 ], "confidence_level": 0.95, "mean": 0.00023232631397515635, "replicates": 8, "sample_sd": 0.000566745698579745, "standard_error": 0.00020037486333702236 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00017519087226506528, 0.00010494189432102945 ], "confidence_level": 0.95, "mean": -2.3416325981345278e-05, "replicates": 8, "sample_sd": 0.00020430360985312392, "standard_error": 7.223223397401733e-05 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0006372243902011687 ], "confidence_level": 0.95, "mean": 0.00023232631397515635, "replicates": 8, "sample_sd": 0.000566745698579745, "standard_error": 0.00020037486333702236 } } }, "discount": { "bootstrap_interval": [ 0.9182108774023049, 1.0 ], "confidence_level": 0.95, "mean": 0.9665887721767087, "replicates": 8, "sample_sd": 0.06231086507856628, "standard_error": 0.022030217619327126 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8524435130225698, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1618358211160297, 0.16370849377999655 ], "confidence_level": 0.95, "mean": 0.16281109900952417, "replicates": 8, "sample_sd": 0.0014467825566713533, "standard_error": 0.0005115148783623621 }, "log_loss": { "bootstrap_interval": [ 0.5119591245084216, 0.5158032354040708 ], "confidence_level": 0.95, "mean": 0.5139799754544809, "replicates": 8, "sample_sd": 0.003007777774358126, "standard_error": 0.001063410030275406 } }, "independent": { "brier": { "bootstrap_interval": [ 0.038842189670345986, 0.04506075511012345 ], "confidence_level": 0.95, "mean": 0.041767779272538684, "replicates": 8, "sample_sd": 0.004778932456983267, "standard_error": 0.0016896077735826783 }, "log_loss": { "bootstrap_interval": [ 0.13429838267427907, 0.15031996508896553 ], "confidence_level": 0.95, "mean": 0.1421976410952961, "replicates": 8, "sample_sd": 0.012394418299532375, "standard_error": 0.00438208861423099 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19469156747935068, 0.20086820859666132 ], "confidence_level": 0.95, "mean": 0.19761153359360628, "replicates": 8, "sample_sd": 0.004840249158008265, "standard_error": 0.0017112865011300604 }, "log_loss": { "bootstrap_interval": [ 0.5706063078390642, 0.584760318472861 ], "confidence_level": 0.95, "mean": 0.5772891634083406, "replicates": 8, "sample_sd": 0.01097636370028827, "standard_error": 0.0038807306026218498 } }, "learned": { "brier": { "bootstrap_interval": [ 0.038863292746521755, 0.04490747624014024 ], "confidence_level": 0.95, "mean": 0.04174436294655734, "replicates": 8, "sample_sd": 0.0046552853904102175, "standard_error": 0.0016458919339588645 }, "log_loss": { "bootstrap_interval": [ 0.13446445489723124, 0.15059143814130965 ], "confidence_level": 0.95, "mean": 0.14242996740927125, "replicates": 8, "sample_sd": 0.01248719599740342, "standard_error": 0.0044148904838847355 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.038818607939919855, 0.045059004508440276 ], "confidence_level": 0.95, "mean": 0.041767779272538684, "replicates": 8, "sample_sd": 0.004778932456983267, "standard_error": 0.0016896077735826783 }, "log_loss": { "bootstrap_interval": [ 0.13429813880516142, 0.15037323548910217 ], "confidence_level": 0.95, "mean": 0.1421976410952961, "replicates": 8, "sample_sd": 0.012394418299532375, "standard_error": 0.00438208861423099 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05704627833027812, -0.05550492226949362 ], "confidence_level": 0.95, "mean": -0.056256649177576536, "replicates": 8, "sample_sd": 0.001187644468287456, "standard_error": 0.00041989572858237584 }, "log_loss": { "bootstrap_interval": [ -0.2604160409274086, -0.2566198399267165 ], "confidence_level": 0.95, "mean": -0.25849375858249696, "replicates": 8, "sample_sd": 0.002897329388907144, "standard_error": 0.0010243606291136585 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 5.3200726554246636e-05 ], "confidence_level": 0.95, "mean": 1.773357551808221e-05, "replicates": 8, "sample_sd": 5.01581260140787e-05, "standard_error": 1.773357551808221e-05 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0005313471338945387 ], "confidence_level": 0.95, "mean": 0.00017711571129817958, "replicates": 8, "sample_sd": 0.0005009588820544863, "standard_error": 0.00017711571129817955 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 5.3200726554246636e-05 ], "confidence_level": 0.95, "mean": 1.773357551808221e-05, "replicates": 8, "sample_sd": 5.01581260140787e-05, "standard_error": 1.773357551808221e-05 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0005313471338945387 ], "confidence_level": 0.95, "mean": 0.00017711571129817958, "replicates": 8, "sample_sd": 0.0005009588820544863, "standard_error": 0.00017711571129817955 } } }, "discount": { "bootstrap_interval": [ 0.8433323226414909, 1.0 ], "confidence_level": 0.95, "mean": 0.947777440880497, "replicates": 8, "sample_sd": 0.14770770273726386, "standard_error": 0.05222255911950302 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.5822195270439758, "discount_one_count": 7, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05629893230176149, 0.05748470541390852 ], "confidence_level": 0.95, "mean": 0.05687773159600846, "replicates": 8, "sample_sd": 0.0009104659627992953, "standard_error": 0.0003218983281674603 }, "log_loss": { "bootstrap_interval": [ 0.2591218660703789, 0.26211945499985 ], "confidence_level": 0.95, "mean": 0.2606102278869234, "replicates": 8, "sample_sd": 0.002342137389192725, "standard_error": 0.000828070615184366 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0003032144573710018, 0.0009232594472686872 ], "confidence_level": 0.95, "mean": 0.0006033488429138398, "replicates": 8, "sample_sd": 0.000481452489609125, "standard_error": 0.00017021916011087903 }, "log_loss": { "bootstrap_interval": [ 0.0011625104004781585, 0.0027812663910969306 ], "confidence_level": 0.95, "mean": 0.0019393535931282885, "replicates": 8, "sample_sd": 0.001267897166695999, "standard_error": 0.0004482693422089757 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1067681040880245, 0.11306194472232331 ], "confidence_level": 0.95, "mean": 0.10971962570460989, "replicates": 8, "sample_sd": 0.004901062078073617, "standard_error": 0.0017327871152110436 }, "log_loss": { "bootstrap_interval": [ 0.33981482678660757, 0.3558594984134751 ], "confidence_level": 0.95, "mean": 0.3478498156750285, "replicates": 8, "sample_sd": 0.01248682896093513, "standard_error": 0.0044147607168969006 } }, "learned": { "brier": { "bootstrap_interval": [ 0.00032742333062283155, 0.0009297424540378638 ], "confidence_level": 0.95, "mean": 0.000621082418431922, "replicates": 8, "sample_sd": 0.00046746993341050826, "standard_error": 0.0001652755799576971 }, "log_loss": { "bootstrap_interval": [ 0.0013616248735814367, 0.002900957085178943 ], "confidence_level": 0.95, "mean": 0.002116469304426468, "replicates": 8, "sample_sd": 0.0011997735227158102, "standard_error": 0.00042418399690021083 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.00030321994899096206, 0.0009234791120670995 ], "confidence_level": 0.95, "mean": 0.0006033488429138398, "replicates": 8, "sample_sd": 0.000481452489609125, "standard_error": 0.00017021916011087903 }, "log_loss": { "bootstrap_interval": [ 0.001136981810219001, 0.0027774111461571827 ], "confidence_level": 0.95, "mean": 0.0019393535931282885, "replicates": 8, "sample_sd": 0.001267897166695999, "standard_error": 0.0004482693422089757 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.1-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07373294533958473, -0.06972160512211639 ], "confidence_level": 0.95, "mean": -0.07181104276858709, "replicates": 8, "sample_sd": 0.0031010567305469502, "standard_error": 0.0010963891215069663 }, "log_loss": { "bootstrap_interval": [ -0.22010478504814113, -0.20987672947297728 ], "confidence_level": 0.95, "mean": -0.21507597794406053, "replicates": 8, "sample_sd": 0.0078687116433509, "standard_error": 0.002782009681107482 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.008341397375123507, -0.005737892073319099 ], "confidence_level": 0.95, "mean": -0.007088221532055468, "replicates": 8, "sample_sd": 0.001986392993489835, "standard_error": 0.000702295977899054 }, "log_loss": { "bootstrap_interval": [ -0.07500665388091132, -0.06452031573089746 ], "confidence_level": 0.95, "mean": -0.06975223219553843, "replicates": 8, "sample_sd": 0.007988664159103003, "standard_error": 0.0028244192997618307 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -6.335019835090015e-05, 0.00030336834392387114 ], "confidence_level": 0.95, "mean": 8.235317307155737e-05, "replicates": 8, "sample_sd": 0.000301321918793631, "standard_error": 0.00010653338604955932 }, "log_loss": { "bootstrap_interval": [ -0.00023131785618611678, 0.0015000409089771306 ], "confidence_level": 0.95, "mean": 0.00041849586955690354, "replicates": 8, "sample_sd": 0.0014879218022586342, "standard_error": 0.0005260597981261947 } } }, "discount": { "bootstrap_interval": [ 0.5154104601432795, 0.5436395504625746 ], "confidence_level": 0.95, "mean": 0.5274545550028057, "replicates": 8, "sample_sd": 0.02249141548850426, "standard_error": 0.007951916205202752 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.577936363409385, "discount_min": 0.5085948092989907, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16605111674282808, 0.16768582271360702 ], "confidence_level": 0.95, "mean": 0.16690144862749365, "replicates": 8, "sample_sd": 0.001280378747810538, "standard_error": 0.0004526822475319859 }, "log_loss": { "bootstrap_interval": [ 0.5183761478795255, 0.521901232149272 ], "confidence_level": 0.95, "mean": 0.5202355443986633, "replicates": 8, "sample_sd": 0.002773546551574409, "standard_error": 0.0009805967872774145 } }, "independent": { "brier": { "bootstrap_interval": [ 0.099216021069144, 0.10542856180989141 ], "confidence_level": 0.95, "mean": 0.102178627390962, "replicates": 8, "sample_sd": 0.004836157793706167, "standard_error": 0.0017098399854089015 }, "log_loss": { "bootstrap_interval": [ 0.36405103103938125, 0.38581277186439156 ], "confidence_level": 0.95, "mean": 0.3749117986501411, "replicates": 8, "sample_sd": 0.01697326748579239, "standard_error": 0.00600095626904847 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1958352849012146, 0.2007082714588063 ], "confidence_level": 0.95, "mean": 0.1981676486512557, "replicates": 8, "sample_sd": 0.0038049404201851056, "standard_error": 0.0013452495865618396 }, "log_loss": { "bootstrap_interval": [ 0.5729044166061259, 0.5842756269773867 ], "confidence_level": 0.95, "mean": 0.578515600100781, "replicates": 8, "sample_sd": 0.008863064931613176, "standard_error": 0.00313356665762018 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09304552483761114, 0.09736624984275659 ], "confidence_level": 0.95, "mean": 0.09509040585890655, "replicates": 8, "sample_sd": 0.0033613500333756785, "standard_error": 0.001188416701270785 }, "log_loss": { "bootstrap_interval": [ 0.2988902151283004, 0.3110920418384244 ], "confidence_level": 0.95, "mean": 0.3051595664546027, "replicates": 8, "sample_sd": 0.009411271647050794, "standard_error": 0.003327387000609152 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.09292810509765759, 0.09719963322620884 ], "confidence_level": 0.95, "mean": 0.09500805268583498, "replicates": 8, "sample_sd": 0.0032970516698490183, "standard_error": 0.0011656837968363355 }, "log_loss": { "bootstrap_interval": [ 0.29887579130080466, 0.31017117814590606 ], "confidence_level": 0.95, "mean": 0.3047410705850458, "replicates": 8, "sample_sd": 0.008942564375285648, "standard_error": 0.003161673955480862 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5263157894736842 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5263157894736842, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.1-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05350174248500349, -0.05207206669398018 ], "confidence_level": 0.95, "mean": -0.05276684063409245, "replicates": 8, "sample_sd": 0.0011090925780479855, "standard_error": 0.00039212344145070035 }, "log_loss": { "bootstrap_interval": [ -0.23585010779968585, -0.22817525171391265 ], "confidence_level": 0.95, "mean": -0.2321797534815687, "replicates": 8, "sample_sd": 0.005959433973073934, "standard_error": 0.0021069780871970337 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0006147206489830698, -5.9838946286173544e-05 ], "confidence_level": 0.95, "mean": -0.0003098106624834894, "replicates": 8, "sample_sd": 0.000426122818670701, "standard_error": 0.00015065716735018913 }, "log_loss": { "bootstrap_interval": [ -0.00912104640116235, -0.0020297320207664014 ], "confidence_level": 0.95, "mean": -0.005477161549485821, "replicates": 8, "sample_sd": 0.0055420304618255135, "standard_error": 0.001959403660549617 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00013977198881639334, 0.000610450163982459 ], "confidence_level": 0.95, "mean": 0.0003610490089345158, "replicates": 8, "sample_sd": 0.00036345094105554417, "standard_error": 0.00012849931252450372 }, "log_loss": { "bootstrap_interval": [ 0.0006597286524937618, 0.005319952387259814 ], "confidence_level": 0.95, "mean": 0.002743036299936792, "replicates": 8, "sample_sd": 0.003651483394861231, "standard_error": 0.001290994334948226 } } }, "discount": { "bootstrap_interval": [ 0.5790783514892485, 0.8718039819536219 ], "confidence_level": 0.95, "mean": 0.723620528262432, "replicates": 8, "sample_sd": 0.22155630917650224, "standard_error": 0.07833198431668402 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.42698691018929014, "discount_one_count": 2, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06180803601971522, 0.06284438553735142 ], "confidence_level": 0.95, "mean": 0.062360434667395236, "replicates": 8, "sample_sd": 0.0008068711052350663, "standard_error": 0.00028527201502759987 }, "log_loss": { "bootstrap_interval": [ 0.26496376260217297, 0.2679394890220298 ], "confidence_level": 0.95, "mean": 0.2665896903056183, "replicates": 8, "sample_sd": 0.002325426030165785, "standard_error": 0.0008221622575389697 } }, "independent": { "brier": { "bootstrap_interval": [ 0.008748755831669973, 0.011013162969408637 ], "confidence_level": 0.95, "mean": 0.009903404695786277, "replicates": 8, "sample_sd": 0.0017760211358197866, "standard_error": 0.0006279182943344026 }, "log_loss": { "bootstrap_interval": [ 0.03302607491949893, 0.046026688257303716 ], "confidence_level": 0.95, "mean": 0.039887098373535455, "replicates": 8, "sample_sd": 0.009967577024127924, "standard_error": 0.003524070652880041 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1086719933186358, 0.11371069609037722 ], "confidence_level": 0.95, "mean": 0.11112646541261903, "replicates": 8, "sample_sd": 0.003931336936424466, "standard_error": 0.0013899375034374436 }, "log_loss": { "bootstrap_interval": [ 0.34395499519011896, 0.35751444279495037 ], "confidence_level": 0.95, "mean": 0.3507030921738555, "replicates": 8, "sample_sd": 0.010442164998598529, "standard_error": 0.0036918628403889175 } }, "learned": { "brier": { "bootstrap_interval": [ 0.00856359999223165, 0.010535983428282625 ], "confidence_level": 0.95, "mean": 0.009593594033302786, "replicates": 8, "sample_sd": 0.0015338876626392758, "standard_error": 0.0005423111839153075 }, "log_loss": { "bootstrap_interval": [ 0.030025211931977366, 0.038871427623412134 ], "confidence_level": 0.95, "mean": 0.034409936824049636, "replicates": 8, "sample_sd": 0.0069272997941583125, "standard_error": 0.0024491703298807586 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.008262463746184827, 0.01013843359692926 ], "confidence_level": 0.95, "mean": 0.009232545024368271, "replicates": 8, "sample_sd": 0.0014439842446544015, "standard_error": 0.000510525525660831 }, "log_loss": { "bootstrap_interval": [ 0.02831401602236672, 0.03487425600196472 ], "confidence_level": 0.95, "mean": 0.03166690052411285, "replicates": 8, "sample_sd": 0.0051385155872346825, "standard_error": 0.001816739608483209 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5263157894736842 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5263157894736842, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.25-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.036769842529327475, -0.03398611644718294 ], "confidence_level": 0.95, "mean": -0.03538377860013598, "replicates": 8, "sample_sd": 0.0021304818530564857, "standard_error": 0.0007532390827455613 }, "log_loss": { "bootstrap_interval": [ -0.10851235050665038, -0.09875700363884757 ], "confidence_level": 0.95, "mean": -0.10364567727249098, "replicates": 8, "sample_sd": 0.0076146099577881505, "standard_error": 0.0026921711686213054 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.02650869049044015, -0.021767797300843657 ], "confidence_level": 0.95, "mean": -0.024265952698330664, "replicates": 8, "sample_sd": 0.00361119668828072, "standard_error": 0.00127675083324085 }, "log_loss": { "bootstrap_interval": [ -0.3119301419138621, -0.2783567975451155 ], "confidence_level": 0.95, "mean": -0.29469386483422993, "replicates": 8, "sample_sd": 0.02631802591746827, "standard_error": 0.00930482729684256 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.061178596584311e-05, 0.000294642297993981 ], "confidence_level": 0.95, "mean": 8.701961637523625e-05, "replicates": 8, "sample_sd": 0.00030041617584723116, "standard_error": 0.00010621315755985372 }, "log_loss": { "bootstrap_interval": [ -0.0003121586338596817, 0.002626645189534962 ], "confidence_level": 0.95, "mean": 0.0008900290149320284, "replicates": 8, "sample_sd": 0.00238735144362288, "standard_error": 0.0008440561974306161 } } }, "discount": { "bootstrap_interval": [ 0.29824964936747783, 0.328024942807183 ], "confidence_level": 0.95, "mean": 0.31201870260610953, "replicates": 8, "sample_sd": 0.02343116216926035, "standard_error": 0.008284166830482845 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3599374193538418, "discount_min": 0.28598956979646517, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17163245714979936, 0.1735298333555623 ], "confidence_level": 0.95, "mean": 0.17266820183338263, "replicates": 8, "sample_sd": 0.0014522297380130025, "standard_error": 0.0005134407477948786 }, "log_loss": { "bootstrap_interval": [ 0.5272438572732918, 0.5314323791529459 ], "confidence_level": 0.95, "mean": 0.5295245785572052, "replicates": 8, "sample_sd": 0.0032181725566570885, "standard_error": 0.001137795818920338 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15780277261621872, 0.16525686355155084 ], "confidence_level": 0.95, "mean": 0.1615503759315773, "replicates": 8, "sample_sd": 0.005740660482049084, "standard_error": 0.0020296299776732707 }, "log_loss": { "bootstrap_interval": [ 0.6991353606884592, 0.7434739526055433 ], "confidence_level": 0.95, "mean": 0.7205727661189442, "replicates": 8, "sample_sd": 0.03443626418486407, "standard_error": 0.01217505796192441 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19668615943194473, 0.20072318072071454 ], "confidence_level": 0.95, "mean": 0.19864580941794724, "replicates": 8, "sample_sd": 0.0030921164901485903, "standard_error": 0.0010932282692014072 }, "log_loss": { "bootstrap_interval": [ 0.5750566706770291, 0.5844625978355317 ], "confidence_level": 0.95, "mean": 0.579797825332825, "replicates": 8, "sample_sd": 0.007299326707341479, "standard_error": 0.0025807017064286165 } }, "learned": { "brier": { "bootstrap_interval": [ 0.13531401935935158, 0.139151942696158 ], "confidence_level": 0.95, "mean": 0.13728442323324666, "replicates": 8, "sample_sd": 0.002943021002620032, "standard_error": 0.0010405150540635283 }, "log_loss": { "bootstrap_interval": [ 0.41924816848562135, 0.43212383282536937 ], "confidence_level": 0.95, "mean": 0.42587890128471423, "replicates": 8, "sample_sd": 0.009870776686588884, "standard_error": 0.0034898465653325403 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.13522285982909535, 0.13898159176452163 ], "confidence_level": 0.95, "mean": 0.1371974036168714, "replicates": 8, "sample_sd": 0.0029257370886365797, "standard_error": 0.0010344042676719563 }, "log_loss": { "bootstrap_interval": [ 0.41894968765545176, 0.43049162295918564 ], "confidence_level": 0.95, "mean": 0.4249888722697822, "replicates": 8, "sample_sd": 0.009008749232015747, "standard_error": 0.003185073835983718 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3076923076923077 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3076923076923077, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.25-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.038592672187309045, -0.035787376321833944 ], "confidence_level": 0.95, "mean": -0.03721643348702066, "replicates": 8, "sample_sd": 0.0021605617561146, "standard_error": 0.0007638739344604746 }, "log_loss": { "bootstrap_interval": [ -0.16807056831812678, -0.15835136294078056 ], "confidence_level": 0.95, "mean": -0.163239018540217, "replicates": 8, "sample_sd": 0.0075481681892075915, "standard_error": 0.0026686804560626352 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.006025519044681983, -0.004138568292768537 ], "confidence_level": 0.95, "mean": -0.005031963377636955, "replicates": 8, "sample_sd": 0.0014641545054982546, "standard_error": 0.0005176567897713259 }, "log_loss": { "bootstrap_interval": [ -0.1236765826227343, -0.09683360810311357 ], "confidence_level": 0.95, "mean": -0.11010871260994745, "replicates": 8, "sample_sd": 0.021078383033085468, "standard_error": 0.0074523337895711006 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00010457968289665854, 7.392889279551939e-05 ], "confidence_level": 0.95, "mean": -5.534082667710532e-06, "replicates": 8, "sample_sd": 0.0001380749765188051, "standard_error": 4.88168761043102e-05 }, "log_loss": { "bootstrap_interval": [ -0.00036480400688854647, 0.00033408360949357084 ], "confidence_level": 0.95, "mean": -1.5089538490636878e-05, "replicates": 8, "sample_sd": 0.0005429862274258345, "standard_error": 0.00019197462175185424 } } }, "discount": { "bootstrap_interval": [ 0.2932258672562194, 0.3239625510528707 ], "confidence_level": 0.95, "mean": 0.3086459419239316, "replicates": 8, "sample_sd": 0.02410423416249534, "standard_error": 0.008522133715804446 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.34010357507219935, "discount_min": 0.27158600123376786, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06957256593392673, 0.0711509681956031 ], "confidence_level": 0.95, "mean": 0.07041767276547645, "replicates": 8, "sample_sd": 0.0012147919162236087, "standard_error": 0.000429493800846157 }, "log_loss": { "bootstrap_interval": [ 0.27376327783400856, 0.27784857474026564 ], "confidence_level": 0.95, "mean": 0.27602107693331934, "replicates": 8, "sample_sd": 0.0031774076631630784, "standard_error": 0.0011233832526083571 } }, "independent": { "brier": { "bootstrap_interval": [ 0.03563173372986968, 0.04098599379604718 ], "confidence_level": 0.95, "mean": 0.03823320265609273, "replicates": 8, "sample_sd": 0.0041139818094213815, "standard_error": 0.0014545122175599807 }, "log_loss": { "bootstrap_interval": [ 0.20364894406015344, 0.24262767407036798 ], "confidence_level": 0.95, "mean": 0.22289077100304977, "replicates": 8, "sample_sd": 0.030461541549906528, "standard_error": 0.01076978129766734 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11035835720060409, 0.11427669137905694 ], "confidence_level": 0.95, "mean": 0.11236518873898231, "replicates": 8, "sample_sd": 0.003048391672834064, "standard_error": 0.001077769211786785 }, "log_loss": { "bootstrap_interval": [ 0.34767122505469306, 0.3591648173865697 ], "confidence_level": 0.95, "mean": 0.3538358241699904, "replicates": 8, "sample_sd": 0.00876608933492542, "standard_error": 0.003099280606606418 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0314827903607419, 0.03513652318643168 ], "confidence_level": 0.95, "mean": 0.03320123927845578, "replicates": 8, "sample_sd": 0.002853147211278027, "standard_error": 0.00100873987040909 }, "log_loss": { "bootstrap_interval": [ 0.10668215035775051, 0.11892822139855683 ], "confidence_level": 0.95, "mean": 0.11278205839310232, "replicates": 8, "sample_sd": 0.009614693732915215, "standard_error": 0.003399307568788074 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.031486185495889725, 0.03513068119196525 ], "confidence_level": 0.95, "mean": 0.03320677336112349, "replicates": 8, "sample_sd": 0.002879060866029009, "standard_error": 0.0010179017309089632 }, "log_loss": { "bootstrap_interval": [ 0.10666688470689824, 0.11906860529030652 ], "confidence_level": 0.95, "mean": 0.11279714793159296, "replicates": 8, "sample_sd": 0.009727570081859298, "standard_error": 0.0034392153846750442 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3076923076923077 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3076923076923077, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.5-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.01095465032838754, -0.008896550273369878 ], "confidence_level": 0.95, "mean": -0.009902369293189266, "replicates": 8, "sample_sd": 0.001577704948862053, "standard_error": 0.0005578029340259664 }, "log_loss": { "bootstrap_interval": [ -0.03271566777922095, -0.022909244784113506 ], "confidence_level": 0.95, "mean": -0.028202231838873715, "replicates": 8, "sample_sd": 0.007712788964811112, "standard_error": 0.0027268826894393545 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.049717861408826274, -0.04545541920134178 ], "confidence_level": 0.95, "mean": -0.047682668809456874, "replicates": 8, "sample_sd": 0.0033423043460567062, "standard_error": 0.001181683033942983 }, "log_loss": { "bootstrap_interval": [ -0.7571763571818575, -0.6979260741140141 ], "confidence_level": 0.95, "mean": -0.7259497822106746, "replicates": 8, "sample_sd": 0.04555031146363498, "standard_error": 0.016104467060547814 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.204981831884333e-07, 0.0008912845268844706 ], "confidence_level": 0.95, "mean": 0.00035637623027871823, "replicates": 8, "sample_sd": 0.0007326023363771015, "standard_error": 0.00025901403998267826 }, "log_loss": { "bootstrap_interval": [ 2.5611167904601782e-06, 0.005474220083692 ], "confidence_level": 0.95, "mean": 0.002049115409935534, "replicates": 8, "sample_sd": 0.004727583504255302, "standard_error": 0.0016714531772422925 } } }, "discount": { "bootstrap_interval": [ 0.17311818684795757, 0.2019704260541559 ], "confidence_level": 0.95, "mean": 0.18654671368640638, "replicates": 8, "sample_sd": 0.02230632807824099, "standard_error": 0.007886477923748046 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.23263558918199323, "discount_min": 0.1565032946282408, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18084530213953592, 0.18325826519213737 ], "confidence_level": 0.95, "mean": 0.1821391944170779, "replicates": 8, "sample_sd": 0.0018584227224317008, "standard_error": 0.0006570516546713103 }, "log_loss": { "bootstrap_interval": [ 0.5434911233884998, 0.5490149783220099 ], "confidence_level": 0.95, "mean": 0.546392373030298, "replicates": 8, "sample_sd": 0.00430106502714926, "standard_error": 0.001520656123510772 } }, "independent": { "brier": { "bootstrap_interval": [ 0.21625495010855694, 0.22281180915567986 ], "confidence_level": 0.95, "mean": 0.2199194939333455, "replicates": 8, "sample_sd": 0.005087691231443316, "standard_error": 0.0017987704851684526 }, "log_loss": { "bootstrap_interval": [ 1.2108170002431118, 1.280522361338074 ], "confidence_level": 0.95, "mean": 1.244139923402099, "replicates": 8, "sample_sd": 0.0535740929948685, "standard_error": 0.01894130222629511 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19771080235408817, 0.20076785111364207 ], "confidence_level": 0.95, "mean": 0.19926702311469635, "replicates": 8, "sample_sd": 0.0023311563873632864, "standard_error": 0.0008241882447554569 }, "log_loss": { "bootstrap_interval": [ 0.5780700767161, 0.5850923175254649 ], "confidence_level": 0.95, "mean": 0.581755125408795, "replicates": 8, "sample_sd": 0.005465067828045487, "standard_error": 0.0019321932604277003 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17016481140059286, 0.1740696214464893 ], "confidence_level": 0.95, "mean": 0.17223682512388863, "replicates": 8, "sample_sd": 0.0030719733677861305, "standard_error": 0.0010861065999930243 }, "log_loss": { "bootstrap_interval": [ 0.511477906607744, 0.5250579946488046 ], "confidence_level": 0.95, "mean": 0.5181901411914243, "replicates": 8, "sample_sd": 0.010770538621393072, "standard_error": 0.003807960448109325 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.169943897065984, 0.17357094806662818 ], "confidence_level": 0.95, "mean": 0.17188044889360993, "replicates": 8, "sample_sd": 0.002840120370153532, "standard_error": 0.001004134186560805 }, "log_loss": { "bootstrap_interval": [ 0.5105747461777949, 0.52140278896572 ], "confidence_level": 0.95, "mean": 0.5161410257814888, "replicates": 8, "sample_sd": 0.008341360752625208, "standard_error": 0.002949116376252304 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.18181818181818182 ], "oracle_method": "oracle", "population_optimal_scalar": 0.18181818181818182, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.5-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.016226745052555603, -0.013682049642190457 ], "confidence_level": 0.95, "mean": -0.014950273490198812, "replicates": 8, "sample_sd": 0.001956764554658211, "standard_error": 0.0006918207428921479 }, "log_loss": { "bootstrap_interval": [ -0.07148655334671725, -0.05947206458360907 ], "confidence_level": 0.95, "mean": -0.06553635273516879, "replicates": 8, "sample_sd": 0.009246579184569276, "standard_error": 0.003269159422093656 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.01648871428220522, -0.012933685995237199 ], "confidence_level": 0.95, "mean": -0.014624615600336899, "replicates": 8, "sample_sd": 0.0027345281894151055, "standard_error": 0.0009668017130405964 }, "log_loss": { "bootstrap_interval": [ -0.5067013606015297, -0.43251428330076336 ], "confidence_level": 0.95, "mean": -0.4692825555901411, "replicates": 8, "sample_sd": 0.05759208236025497, "standard_error": 0.020361875989795216 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00012984872979559479, 0.00022823306457545912 ], "confidence_level": 0.95, "mean": 3.584290682755002e-05, "replicates": 8, "sample_sd": 0.00027223113232492614, "standard_error": 9.62482398585238e-05 }, "log_loss": { "bootstrap_interval": [ -0.0007930010840148001, 0.003870011676730619 ], "confidence_level": 0.95, "mean": 0.0010200118453376032, "replicates": 8, "sample_sd": 0.0038878743809661704, "standard_error": 0.0013745711695913147 } } }, "discount": { "bootstrap_interval": [ 0.16813463124910008, 0.1962659989495302 ], "confidence_level": 0.95, "mean": 0.18108670954299944, "replicates": 8, "sample_sd": 0.021469185388177037, "standard_error": 0.007590503287265561 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.22268894319266824, "discount_min": 0.15631833761175823, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08309835977675088, 0.08575805134420547 ], "confidence_level": 0.95, "mean": 0.08440898967928924, "replicates": 8, "sample_sd": 0.0020256501136247116, "standard_error": 0.000716175465827667 }, "log_loss": { "bootstrap_interval": [ 0.29266855165648475, 0.2994032647319469 ], "confidence_level": 0.95, "mean": 0.296146634832802, "replicates": 8, "sample_sd": 0.00518907285577925, "standard_error": 0.0018346143021962754 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08085590963948198, 0.08805837795416742 ], "confidence_level": 0.95, "mean": 0.08408333178942733, "replicates": 8, "sample_sd": 0.005615800457684956, "standard_error": 0.0019854852927097748 }, "log_loss": { "bootstrap_interval": [ 0.6553883210312722, 0.7459087173300628 ], "confidence_level": 0.95, "mean": 0.6998928376877743, "replicates": 8, "sample_sd": 0.0703221952389286, "standard_error": 0.02486265056068538 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11191013008342315, 0.11537443308546913 ], "confidence_level": 0.95, "mean": 0.11374706502855107, "replicates": 8, "sample_sd": 0.0026832453909882235, "standard_error": 0.0009486705057776608 }, "log_loss": { "bootstrap_interval": [ 0.35299020235766276, 0.36231231796595464 ], "confidence_level": 0.95, "mean": 0.3585218932160255, "replicates": 8, "sample_sd": 0.007593125351894876, "standard_error": 0.0026845752133621784 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06731584137031295, 0.0718807515656874 ], "confidence_level": 0.95, "mean": 0.06945871618909043, "replicates": 8, "sample_sd": 0.003538939456886146, "standard_error": 0.0012512040440864156 }, "log_loss": { "bootstrap_interval": [ 0.22238875033800526, 0.2396558205547339 ], "confidence_level": 0.95, "mean": 0.23061028209763323, "replicates": 8, "sample_sd": 0.013486635487593728, "standard_error": 0.004768245704334332 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06721285401766408, 0.07188829944570518 ], "confidence_level": 0.95, "mean": 0.06942287328226288, "replicates": 8, "sample_sd": 0.003650330116524017, "standard_error": 0.001290586589481806 }, "log_loss": { "bootstrap_interval": [ 0.2219717788779202, 0.23790400619412236 ], "confidence_level": 0.95, "mean": 0.22959027025229561, "replicates": 8, "sample_sd": 0.012272100230778498, "standard_error": 0.004338842646292235 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.18181818181818182 ], "oracle_method": "oracle", "population_optimal_scalar": 0.18181818181818182, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.75-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.001890034177762026, -0.00012429341680786547 ], "confidence_level": 0.95, "mean": -0.0011012242192723447, "replicates": 8, "sample_sd": 0.0013774533273343305, "standard_error": 0.0004870032942630391 }, "log_loss": { "bootstrap_interval": [ -0.0056686682134545075, 0.002589435366908935 ], "confidence_level": 0.95, "mean": -0.0022933805157785564, "replicates": 8, "sample_sd": 0.006607434568987964, "standard_error": 0.002336080894988901 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06725459657664222, -0.06305091526578756 ], "confidence_level": 0.95, "mean": -0.06545733419761962, "replicates": 8, "sample_sd": 0.003363105565758371, "standard_error": 0.0011890373756969822 }, "log_loss": { "bootstrap_interval": [ -1.1854881979501235, -1.1059299111221537 ], "confidence_level": 0.95, "mean": -1.1439043494639205, "replicates": 8, "sample_sd": 0.06077250892764386, "standard_error": 0.021486326586228486 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.2092129992048486e-05, 0.0013022476376903516 ], "confidence_level": 0.95, "mean": 0.0005382049200009092, "replicates": 8, "sample_sd": 0.0010299384090484821, "standard_error": 0.0003641382166213329 }, "log_loss": { "bootstrap_interval": [ 0.0001324445042578809, 0.006310376306895894 ], "confidence_level": 0.95, "mean": 0.00243762368727031, "replicates": 8, "sample_sd": 0.005266343030531783, "standard_error": 0.0018619334344717685 } } }, "discount": { "bootstrap_interval": [ 0.12141434558518524, 0.14553704610671364 ], "confidence_level": 0.95, "mean": 0.13271922565587185, "replicates": 8, "sample_sd": 0.018523800284351744, "standard_error": 0.006549152397205207 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.17029138440835123, "discount_min": 0.10561685031970136, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1900367909421344, 0.19298702102585819 ], "confidence_level": 0.95, "mean": 0.19155512339645234, "replicates": 8, "sample_sd": 0.00227417079688591, "standard_error": 0.0008040407960272207 }, "log_loss": { "bootstrap_interval": [ 0.5616686266997062, 0.5690900706346338 ], "confidence_level": 0.95, "mean": 0.5654273880577126, "replicates": 8, "sample_sd": 0.00557961402610496, "standard_error": 0.0019726914571311957 } }, "independent": { "brier": { "bootstrap_interval": [ 0.25168510720947335, 0.2587854548767632 ], "confidence_level": 0.95, "mean": 0.2559112333747996, "replicates": 8, "sample_sd": 0.005694432871265361, "standard_error": 0.0020132860491416595 }, "log_loss": { "bootstrap_interval": [ 1.6647218013170246, 1.7554458563841466 ], "confidence_level": 0.95, "mean": 1.7070383570058545, "replicates": 8, "sample_sd": 0.06915755308885363, "standard_error": 0.02445088737969853 } }, "individual": { "brier": { "bootstrap_interval": [ 0.198561907715546, 0.20111495827926346 ], "confidence_level": 0.95, "mean": 0.19980328185793006, "replicates": 8, "sample_sd": 0.001967302994709199, "standard_error": 0.0006955466441037386 }, "log_loss": { "bootstrap_interval": [ 0.5806797486957582, 0.586764117742334 ], "confidence_level": 0.95, "mean": 0.5837083029857186, "replicates": 8, "sample_sd": 0.004731593149177854, "standard_error": 0.0016728708007997361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1883832886466292, 0.19241242099299702 ], "confidence_level": 0.95, "mean": 0.19045389917717997, "replicates": 8, "sample_sd": 0.003177713823772971, "standard_error": 0.0011234914967300507 }, "log_loss": { "bootstrap_interval": [ 0.5567010852958846, 0.5702648257990001 ], "confidence_level": 0.95, "mean": 0.5631340075419341, "replicates": 8, "sample_sd": 0.010446099059098376, "standard_error": 0.0036932537408174374 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18811617693311014, 0.19161391539762465 ], "confidence_level": 0.95, "mean": 0.18991569425717908, "replicates": 8, "sample_sd": 0.0027756751877602006, "standard_error": 0.0009813493738182405 }, "log_loss": { "bootstrap_interval": [ 0.5557228100133876, 0.5656867002154855 ], "confidence_level": 0.95, "mean": 0.5606963838546637, "replicates": 8, "sample_sd": 0.007614396863157996, "standard_error": 0.0026920958282922974 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12903225806451613 ], "oracle_method": "oracle", "population_optimal_scalar": 0.12903225806451613, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho0.75-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0035941329423463506, -0.0023335189960194836 ], "confidence_level": 0.95, "mean": -0.0029341304945995506, "replicates": 8, "sample_sd": 0.0009786055929110087, "standard_error": 0.0003459893254272281 }, "log_loss": { "bootstrap_interval": [ -0.016256037800426074, -0.0028458672529325033 ], "confidence_level": 0.95, "mean": -0.010793155396302885, "replicates": 8, "sample_sd": 0.010835608066570442, "standard_error": 0.003830965971075807 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.02826610517495267, -0.02359126704820867 ], "confidence_level": 0.95, "mean": -0.02593570065039727, "replicates": 8, "sample_sd": 0.003602318534294132, "standard_error": 0.0012736119317966826 }, "log_loss": { "bootstrap_interval": [ -1.0008358002131532, -0.8847088240286948 ], "confidence_level": 0.95, "mean": -0.9403294769859433, "replicates": 8, "sample_sd": 0.09112829232123085, "standard_error": 0.032218716729146155 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -7.996316977112803e-05, 0.0006518929475606384 ], "confidence_level": 0.95, "mean": 0.0002501109622791188, "replicates": 8, "sample_sd": 0.0005699363202988852, "standard_error": 0.00020150291846392495 }, "log_loss": { "bootstrap_interval": [ -0.000489965574865546, 0.009542764006176761 ], "confidence_level": 0.95, "mean": 0.003238038557281067, "replicates": 8, "sample_sd": 0.008618538593391633, "standard_error": 0.003047113541652596 } } }, "discount": { "bootstrap_interval": [ 0.1188936073207214, 0.1470561823945036 ], "confidence_level": 0.95, "mean": 0.13134494663164875, "replicates": 8, "sample_sd": 0.022113132367234924, "standard_error": 0.007818172925073773 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.17720938808951803, "discount_min": 0.1101807700037703, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09769403552587834, 0.10135320231443323 ], "confidence_level": 0.95, "mean": 0.09945728702673771, "replicates": 8, "sample_sd": 0.0028239168428146914, "standard_error": 0.000998405374530587 }, "log_loss": { "bootstrap_interval": [ 0.31993514412648716, 0.3304983772408134 ], "confidence_level": 0.95, "mean": 0.3249691885049578, "replicates": 8, "sample_sd": 0.00815697864206768, "standard_error": 0.0028839274558999464 } }, "independent": { "brier": { "bootstrap_interval": [ 0.11912568407404729, 0.12664543579358878 ], "confidence_level": 0.95, "mean": 0.12245885718253544, "replicates": 8, "sample_sd": 0.005828102744958218, "standard_error": 0.002060545486205944 }, "log_loss": { "bootstrap_interval": [ 1.1895056586086998, 1.324885044418756 ], "confidence_level": 0.95, "mean": 1.254505510094598, "replicates": 8, "sample_sd": 0.10544032667607593, "standard_error": 0.03727878500158906 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11273761152425156, 0.11677794332035625 ], "confidence_level": 0.95, "mean": 0.11469046688997823, "replicates": 8, "sample_sd": 0.003156091016768506, "standard_error": 0.0011158466799994781 }, "log_loss": { "bootstrap_interval": [ 0.35708544108799345, 0.36862515447489014 ], "confidence_level": 0.95, "mean": 0.3628740273563677, "replicates": 8, "sample_sd": 0.008938420049611378, "standard_error": 0.0031602087150870008 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09432566463474591, 0.09887600590250527 ], "confidence_level": 0.95, "mean": 0.09652315653213817, "replicates": 8, "sample_sd": 0.0035824762366900354, "standard_error": 0.0012665966202015935 }, "log_loss": { "bootstrap_interval": [ 0.3043200892364497, 0.3263821273412086 ], "confidence_level": 0.95, "mean": 0.3141760331086549, "replicates": 8, "sample_sd": 0.017155941198042247, "standard_error": 0.006065541179386667 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.09408659429372862, 0.09875969421325238 ], "confidence_level": 0.95, "mean": 0.09627304556985904, "replicates": 8, "sample_sd": 0.003614698436077397, "standard_error": 0.0012779888880473675 }, "log_loss": { "bootstrap_interval": [ 0.3032928120037219, 0.3193660346006869 ], "confidence_level": 0.95, "mean": 0.31093799455137383, "replicates": 8, "sample_sd": 0.012411542655601286, "standard_error": 0.004388142988380879 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12903225806451613 ], "oracle_method": "oracle", "population_optimal_scalar": 0.12903225806451613, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho1-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 8.564003494221373e-05, 0.0016136503890401088 ], "confidence_level": 0.95, "mean": 0.0007045402269900553, "replicates": 8, "sample_sd": 0.0012227781525520922, "standard_error": 0.0004323173617781715 }, "log_loss": { "bootstrap_interval": [ 0.0002899877183496628, 0.006799255965498123 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.08087463560040567, -0.07637596859173887 ], "confidence_level": 0.95, "mean": -0.07855035286762285, "replicates": 8, "sample_sd": 0.003511727930695548, "standard_error": 0.001241583316738512 }, "log_loss": { "bootstrap_interval": [ -1.5819331390431213, -1.4841097402913932 ], "confidence_level": 0.95, "mean": -1.5287820971283497, "replicates": 8, "sample_sd": 0.07587251470084223, "standard_error": 0.026824984825320777 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.564003494221373e-05, 0.0016136503890400982 ], "confidence_level": 0.95, "mean": 0.0007045402269900518, "replicates": 8, "sample_sd": 0.001222778152552083, "standard_error": 0.0004323173617781683 }, "log_loss": { "bootstrap_interval": [ 0.0002899877183496628, 0.006799255965498123 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } } }, "discount": { "bootstrap_interval": [ 0.09311718005454582, 0.11450207822883618 ], "confidence_level": 0.95, "mean": 0.10297594134836385, "replicates": 8, "sample_sd": 0.016659029798790956, "standard_error": 0.0058898564693569255 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.1353887636745708, "discount_min": 0.07813152934213824, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1989918006774526, 0.20254290857825819 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.0027431450244216854, "standard_error": 0.0009698482242733556 }, "log_loss": { "bootstrap_interval": [ 0.5821874344464785, 0.591499124982206 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "independent": { "brier": { "bootstrap_interval": [ 0.27664165027118975, 0.28287034271750916 ], "confidence_level": 0.95, "mean": 0.2799860352849783, "replicates": 8, "sample_sd": 0.004952333536138131, "standard_error": 0.0017509143130504132 }, "log_loss": { "bootstrap_interval": [ 2.0678064359019896, 2.1770414786695493 ], "confidence_level": 0.95, "mean": 2.118147735842108, "replicates": 8, "sample_sd": 0.08469687828102482, "standard_error": 0.02994486848892213 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1989688691734306, 0.20253900544352396 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5821675530333215, 0.5913884721608538 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1994622067101678, 0.2034380932800742 ], "confidence_level": 0.95, "mean": 0.20143568241735543, "replicates": 8, "sample_sd": 0.00314778408765836, "standard_error": 0.001112909737047168 }, "log_loss": { "bootstrap_interval": [ 0.5834502105100372, 0.596141123137039 ], "confidence_level": 0.95, "mean": 0.5893656387137585, "replicates": 8, "sample_sd": 0.009854990310042909, "standard_error": 0.0034842652383795286 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1989747986586022, 0.2025666636329837 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5821677680610418, 0.5914205437931368 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1 ], "oracle_method": "oracle", "population_optimal_scalar": 0.1, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n10-rho1-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 7.848530575306966e-05, 0.0015975088888447262 ], "confidence_level": 0.95, "mean": 0.0007233667431064527, "replicates": 8, "sample_sd": 0.0011947528903027155, "standard_error": 0.0004224089352876387 }, "log_loss": { "bootstrap_interval": [ 0.00011414315819317095, 0.016205512835962922 ], "confidence_level": 0.95, "mean": 0.006085127988558439, "replicates": 8, "sample_sd": 0.013870011259328706, "standard_error": 0.004903789508302546 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.040242816515148806, -0.032687308659205785 ], "confidence_level": 0.95, "mean": -0.03631498357666844, "replicates": 8, "sample_sd": 0.005909017492082502, "standard_error": 0.0020891531694007316 }, "log_loss": { "bootstrap_interval": [ -1.5272256422924766, -1.368257877860741 ], "confidence_level": 0.95, "mean": -1.441535050009702, "replicates": 8, "sample_sd": 0.12230102295822666, "standard_error": 0.04323994133990685 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 7.848530575306611e-05, 0.0015975088888447245 ], "confidence_level": 0.95, "mean": 0.000723366743106451, "replicates": 8, "sample_sd": 0.0011947528903027164, "standard_error": 0.00042240893528763904 }, "log_loss": { "bootstrap_interval": [ 0.00011414315819316401, 0.016205512835962922 ], "confidence_level": 0.95, "mean": 0.006085127988558432, "replicates": 8, "sample_sd": 0.013870011259328708, "standard_error": 0.004903789508302547 } } }, "discount": { "bootstrap_interval": [ 0.09169821149988584, 0.11966786064531879 ], "confidence_level": 0.95, "mean": 0.1040270855649435, "replicates": 8, "sample_sd": 0.021771056501711558, "standard_error": 0.007697230842977858 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.14909187209878377, "discount_min": 0.08570573227193243, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.11363133374510022, 0.11835301145167303 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.3601190385304718, 0.3762037866812055 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1479286659306514, 0.15840743058845788 ], "confidence_level": 0.95, "mean": 0.15286099794616578, "replicates": 8, "sample_sd": 0.008136071776772077, "standard_error": 0.002876535762788009 }, "log_loss": { "bootstrap_interval": [ 1.7298448285083796, 1.9095749431232834 ], "confidence_level": 0.95, "mean": 1.8153629117298808, "replicates": 8, "sample_sd": 0.1396809089965319, "standard_error": 0.04938465897687437 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11363133374510022, 0.1183338136437823 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.3601486658967729, 0.3763576012555263 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11406703707218804, 0.11931729118842294 ], "confidence_level": 0.95, "mean": 0.11654601436949735, "replicates": 8, "sample_sd": 0.004073392697855985, "standard_error": 0.0014401617995448662 }, "log_loss": { "bootstrap_interval": [ 0.36213381150377744, 0.389657469903916 ], "confidence_level": 0.95, "mean": 0.373827861720179, "replicates": 8, "sample_sd": 0.022116164976176075, "standard_error": 0.00781924511424726 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11363133374510023, 0.11832474819776535 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.003638834271105444, "standard_error": 0.0012865221943563336 }, "log_loss": { "bootstrap_interval": [ 0.36022249605145085, 0.3763967888837056 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1 ], "oracle_method": "oracle", "population_optimal_scalar": 0.1, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.019431668706613205, -0.017458754269670876 ], "confidence_level": 0.95, "mean": -0.01842248995141843, "replicates": 8, "sample_sd": 0.0015144305393541887, "standard_error": 0.0005354320520066737 }, "log_loss": { "bootstrap_interval": [ -0.05777436381899012, -0.05128147225111271 ], "confidence_level": 0.95, "mean": -0.05445662228096347, "replicates": 8, "sample_sd": 0.004990860080663707, "standard_error": 0.0017645355034952733 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 5.987362778129418e-05, 0.000680492164748879 ], "confidence_level": 0.95, "mean": 0.0003442340266602874, "replicates": 8, "sample_sd": 0.00048117452298415773, "standard_error": 0.0001701208840681501 }, "log_loss": { "bootstrap_interval": [ 0.0002819641845360349, 0.00306909668784631 ], "confidence_level": 0.95, "mean": 0.0015123836281973405, "replicates": 8, "sample_sd": 0.002202889269231917, "standard_error": 0.0007788389702384833 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.987362778129418e-05, 0.000680492164748879 ], "confidence_level": 0.95, "mean": 0.0003442340266602874, "replicates": 8, "sample_sd": 0.00048117452298415773, "standard_error": 0.0001701208840681501 }, "log_loss": { "bootstrap_interval": [ 0.0002819641845360349, 0.00306909668784631 ], "confidence_level": 0.95, "mean": 0.0015123836281973405, "replicates": 8, "sample_sd": 0.002202889269231917, "standard_error": 0.0007788389702384833 } } }, "discount": { "bootstrap_interval": [ 0.91371878345138, 0.9884506216889211 ], "confidence_level": 0.95, "mean": 0.9539720471479203, "replicates": 8, "sample_sd": 0.058110047099370545, "standard_error": 0.02054500417951729 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8451809169710223, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17499416798066703, 0.17794641041575449 ], "confidence_level": 0.95, "mean": 0.1764219948903957, "replicates": 8, "sample_sd": 0.0023632392549242944, "standard_error": 0.0008355312513616063 }, "log_loss": { "bootstrap_interval": [ 0.5314038253945589, 0.5380932735460829 ], "confidence_level": 0.95, "mean": 0.5346657865136408, "replicates": 8, "sample_sd": 0.005195973837891286, "standard_error": 0.0018370541678204096 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15525198319296335, 0.1601395228087473 ], "confidence_level": 0.95, "mean": 0.15765527091231699, "replicates": 8, "sample_sd": 0.0037972374041909435, "standard_error": 0.0013425261591393095 }, "log_loss": { "bootstrap_interval": [ 0.47242815827251744, 0.48558125472267744 ], "confidence_level": 0.95, "mean": 0.47869678060448007, "replicates": 8, "sample_sd": 0.01033524979586564, "standard_error": 0.0036540626079567375 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19464561271129244, 0.20092217337047286 ], "confidence_level": 0.95, "mean": 0.19761153359360628, "replicates": 8, "sample_sd": 0.004840249158008265, "standard_error": 0.0017112865011300604 }, "log_loss": { "bootstrap_interval": [ 0.5706108442289506, 0.5847032525907262 ], "confidence_level": 0.95, "mean": 0.5772891634083406, "replicates": 8, "sample_sd": 0.01097636370028827, "standard_error": 0.0038807306026218498 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1556266482464334, 0.16032968273761483 ], "confidence_level": 0.95, "mean": 0.15799950493897727, "replicates": 8, "sample_sd": 0.003615854433460037, "standard_error": 0.0012783975948415169 }, "log_loss": { "bootstrap_interval": [ 0.4742422532948977, 0.4864493962497531 ], "confidence_level": 0.95, "mean": 0.4802091642326774, "replicates": 8, "sample_sd": 0.009572981116126127, "standard_error": 0.003384559931691774 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.15525198319296335, 0.1601395228087473 ], "confidence_level": 0.95, "mean": 0.15765527091231699, "replicates": 8, "sample_sd": 0.0037972374041909435, "standard_error": 0.0013425261591393095 }, "log_loss": { "bootstrap_interval": [ 0.4722006817879428, 0.485585039964539 ], "confidence_level": 0.95, "mean": 0.47869678060448007, "replicates": 8, "sample_sd": 0.01033524979586564, "standard_error": 0.0036540626079567375 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.026653407348769125, -0.025243844246547492 ], "confidence_level": 0.95, "mean": -0.025985335881461003, "replicates": 8, "sample_sd": 0.0011080962341759443, "standard_error": 0.00039177118069654335 }, "log_loss": { "bootstrap_interval": [ -0.10795763181121953, -0.10073454724490535 ], "confidence_level": 0.95, "mean": -0.10440732166184392, "replicates": 8, "sample_sd": 0.005679689370122224, "standard_error": 0.0020080734343232874 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0004805174422145538 ], "confidence_level": 0.95, "mean": 0.00020452086993799027, "replicates": 8, "sample_sd": 0.0004083716274176134, "standard_error": 0.00014438117349559033 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0031505423726970044 ], "confidence_level": 0.95, "mean": 0.0011930923096123217, "replicates": 8, "sample_sd": 0.0027479610891280207, "standard_error": 0.000971550960279597 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0004805174422145538 ], "confidence_level": 0.95, "mean": 0.00020452086993799027, "replicates": 8, "sample_sd": 0.0004083716274176134, "standard_error": 0.00014438117349559033 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0031505423726970044 ], "confidence_level": 0.95, "mean": 0.0011930923096123217, "replicates": 8, "sample_sd": 0.0027479610891280207, "standard_error": 0.000971550960279597 } } }, "discount": { "bootstrap_interval": [ 0.9253563022937135, 1.0 ], "confidence_level": 0.95, "mean": 0.9682796011509758, "replicates": 8, "sample_sd": 0.06201933445138564, "standard_error": 0.021927145977625625 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8358600092737121, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07728049400346969, 0.08015136286581322 ], "confidence_level": 0.95, "mean": 0.0786547210477763, "replicates": 8, "sample_sd": 0.0022264577157662473, "standard_error": 0.0007871716744217121 }, "log_loss": { "bootstrap_interval": [ 0.27903578571194876, 0.2863027704026195 ], "confidence_level": 0.95, "mean": 0.28246419563760766, "replicates": 8, "sample_sd": 0.005691821613961133, "standard_error": 0.0020123628302680385 } }, "independent": { "brier": { "bootstrap_interval": [ 0.050637880721712214, 0.054377251364406166 ], "confidence_level": 0.95, "mean": 0.052464864296377305, "replicates": 8, "sample_sd": 0.002885108384326291, "standard_error": 0.001020039851507642 }, "log_loss": { "bootstrap_interval": [ 0.17028270285180153, 0.18397128390843506 ], "confidence_level": 0.95, "mean": 0.1768637816661514, "replicates": 8, "sample_sd": 0.010710790741992497, "standard_error": 0.0037868363827664934 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10677955248554252, 0.1130460147556684 ], "confidence_level": 0.95, "mean": 0.10971962570460989, "replicates": 8, "sample_sd": 0.004901062078073617, "standard_error": 0.0017327871152110436 }, "log_loss": { "bootstrap_interval": [ 0.33998307275493844, 0.3557568169238027 ], "confidence_level": 0.95, "mean": 0.3478498156750285, "replicates": 8, "sample_sd": 0.01248682896093513, "standard_error": 0.0044147607168969006 } }, "learned": { "brier": { "bootstrap_interval": [ 0.05099564749936492, 0.05444705076672916 ], "confidence_level": 0.95, "mean": 0.052669385166315294, "replicates": 8, "sample_sd": 0.0026986654667564055, "standard_error": 0.0009541223258487068 }, "log_loss": { "bootstrap_interval": [ 0.17194876568031406, 0.1847614521398182 ], "confidence_level": 0.95, "mean": 0.17805687397576372, "replicates": 8, "sample_sd": 0.009891937061380744, "standard_error": 0.003497327887586427 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.050684242721407397, 0.0543695203205379 ], "confidence_level": 0.95, "mean": 0.052464864296377305, "replicates": 8, "sample_sd": 0.002885108384326291, "standard_error": 0.001020039851507642 }, "log_loss": { "bootstrap_interval": [ 0.17031082049911342, 0.18404055392981028 ], "confidence_level": 0.95, "mean": 0.1768637816661514, "replicates": 8, "sample_sd": 0.010710790741992497, "standard_error": 0.0037868363827664934 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.1-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.013724471737511377, -0.011840522127933987 ], "confidence_level": 0.95, "mean": -0.012739361717895217, "replicates": 8, "sample_sd": 0.001455062093268809, "standard_error": 0.0005144421365989337 }, "log_loss": { "bootstrap_interval": [ -0.0411543122693766, -0.03575525324479917 ], "confidence_level": 0.95, "mean": -0.03817600800446433, "replicates": 8, "sample_sd": 0.004185509046594207, "standard_error": 0.0014798009147822025 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 5.5796211218588077e-05, 0.0007143300034534911 ], "confidence_level": 0.95, "mean": 0.00036028246909188336, "replicates": 8, "sample_sd": 0.0005118077261952105, "standard_error": 0.00018095135692815055 }, "log_loss": { "bootstrap_interval": [ 0.00018557888577406867, 0.0036151315386683744 ], "confidence_level": 0.95, "mean": 0.001754321785014834, "replicates": 8, "sample_sd": 0.0027727116446533396, "standard_error": 0.0009803016031046405 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00021403333750344422, 0.0008842572918653242 ], "confidence_level": 0.95, "mean": 0.0005303095031998807, "replicates": 8, "sample_sd": 0.0005158059108456731, "standard_error": 0.0001823649286675396 }, "log_loss": { "bootstrap_interval": [ 0.0005649824761414235, 0.003890620139785232 ], "confidence_level": 0.95, "mean": 0.002140995453300071, "replicates": 8, "sample_sd": 0.002597264720412017, "standard_error": 0.0009182717481699598 } } }, "discount": { "bootstrap_interval": [ 0.8160663677047103, 0.9665651337833532 ], "confidence_level": 0.95, "mean": 0.894831177857944, "replicates": 8, "sample_sd": 0.11788970723619936, "standard_error": 0.04168030570940668 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.735963986662139, "discount_one_count": 2, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1780661055745905, 0.18033970986047682 ], "confidence_level": 0.95, "mean": 0.17918709955733636, "replicates": 8, "sample_sd": 0.0017344341829043809, "standard_error": 0.0006132150861267181 }, "log_loss": { "bootstrap_interval": [ 0.5373254514150416, 0.542326844218717 ], "confidence_level": 0.95, "mean": 0.5397739177744149, "replicates": 8, "sample_sd": 0.0038441366995692356, "standard_error": 0.00135910756403674 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1640818240738803, 0.16802715392842646 ], "confidence_level": 0.95, "mean": 0.16608745537034925, "replicates": 8, "sample_sd": 0.0030904865362931084, "standard_error": 0.001092651993489291 }, "log_loss": { "bootstrap_interval": [ 0.49464749072963204, 0.5049181969549701 ], "confidence_level": 0.95, "mean": 0.49984358798493567, "replicates": 8, "sample_sd": 0.00794386407057509, "standard_error": 0.002808580076563908 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19579430049963564, 0.200686032573119 ], "confidence_level": 0.95, "mean": 0.1981676486512557, "replicates": 8, "sample_sd": 0.0038049404201851056, "standard_error": 0.0013452495865618396 }, "log_loss": { "bootstrap_interval": [ 0.5727953078818249, 0.5841904168665589 ], "confidence_level": 0.95, "mean": 0.578515600100781, "replicates": 8, "sample_sd": 0.008863064931613176, "standard_error": 0.00313356665762018 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16446748537795142, 0.16827431377773944 ], "confidence_level": 0.95, "mean": 0.16644773783944114, "replicates": 8, "sample_sd": 0.0029562899511240727, "standard_error": 0.0010452063357967395 }, "log_loss": { "bootstrap_interval": [ 0.4966690971656531, 0.5059453171196009 ], "confidence_level": 0.95, "mean": 0.5015979097699504, "replicates": 8, "sample_sd": 0.0072876635622698095, "standard_error": 0.0025765781619435466 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1640705571659919, 0.16778300250068734 ], "confidence_level": 0.95, "mean": 0.16591742833624126, "replicates": 8, "sample_sd": 0.002888154051247776, "standard_error": 0.0010211166573743508 }, "log_loss": { "bootstrap_interval": [ 0.4948082078207889, 0.5042127688681594 ], "confidence_level": 0.95, "mean": 0.4994569143166504, "replicates": 8, "sample_sd": 0.007248810723886098, "standard_error": 0.002562841609198813 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.9090909090909091 ], "oracle_method": "oracle", "population_optimal_scalar": 0.9090909090909091, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.1-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.02137292729984796, -0.019471600073890268 ], "confidence_level": 0.95, "mean": -0.02046457007757601, "replicates": 8, "sample_sd": 0.0014680042330484063, "standard_error": 0.0005190178739995424 }, "log_loss": { "bootstrap_interval": [ -0.08653771762872722, -0.0786992636841204 ], "confidence_level": 0.95, "mean": -0.0823641703835036, "replicates": 8, "sample_sd": 0.006050671283564532, "standard_error": 0.002139235347669596 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -4.336347096934017e-05, 0.0003071076577709506 ], "confidence_level": 0.95, "mean": 0.00010457791506121496, "replicates": 8, "sample_sd": 0.0002780789080514925, "standard_error": 9.831574079408039e-05 }, "log_loss": { "bootstrap_interval": [ -0.0006095074550877197, 0.0022959630932945993 ], "confidence_level": 0.95, "mean": 0.0006903946595830622, "replicates": 8, "sample_sd": 0.002281776670687928, "standard_error": 0.0008067298784983488 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.024605737978259e-05, 0.00030872083008710616 ], "confidence_level": 0.95, "mean": 0.00010947712100568656, "replicates": 8, "sample_sd": 0.00027281608575799966, "standard_error": 9.645505212812611e-05 }, "log_loss": { "bootstrap_interval": [ -0.0003174614932417899, 0.0022627448745295236 ], "confidence_level": 0.95, "mean": 0.0008106032703560709, "replicates": 8, "sample_sd": 0.002028560186679757, "standard_error": 0.0007172043320231525 } } }, "discount": { "bootstrap_interval": [ 0.8615982006734855, 0.9680685183174548 ], "confidence_level": 0.95, "mean": 0.9186814535509024, "replicates": 8, "sample_sd": 0.08357714404025765, "standard_error": 0.029548982651535514 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7625545384076449, "discount_one_count": 2, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08137088194564192, 0.08378331003608369 ], "confidence_level": 0.95, "mean": 0.08257567666569739, "replicates": 8, "sample_sd": 0.001897593779781507, "standard_error": 0.0006709007148104579 }, "log_loss": { "bootstrap_interval": [ 0.28608654562363245, 0.29208356609667985 ], "confidence_level": 0.95, "mean": 0.2892025107513427, "replicates": 8, "sample_sd": 0.004590655989604773, "standard_error": 0.001623041990172088 } }, "independent": { "brier": { "bootstrap_interval": [ 0.060140921092582386, 0.06388546089778224 ], "confidence_level": 0.95, "mean": 0.06200652867306017, "replicates": 8, "sample_sd": 0.0028850877830832364, "standard_error": 0.0010200325678683098 }, "log_loss": { "bootstrap_interval": [ 0.19948369689764578, 0.21259063445493498 ], "confidence_level": 0.95, "mean": 0.206147945708256, "replicates": 8, "sample_sd": 0.010196700046733419, "standard_error": 0.003605077874385193 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10873714426167763, 0.11376123062279223 ], "confidence_level": 0.95, "mean": 0.11112646541261903, "replicates": 8, "sample_sd": 0.003931336936424466, "standard_error": 0.0013899375034374436 }, "log_loss": { "bootstrap_interval": [ 0.34396860978512644, 0.3574650856757817 ], "confidence_level": 0.95, "mean": 0.3507030921738555, "replicates": 8, "sample_sd": 0.010442164998598529, "standard_error": 0.0036918628403889175 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06029442826442321, 0.06390837590724716 ], "confidence_level": 0.95, "mean": 0.062111106588121384, "replicates": 8, "sample_sd": 0.002846067836699818, "standard_error": 0.0010062369335236843 }, "log_loss": { "bootstrap_interval": [ 0.20030222808582743, 0.2126864739657092 ], "confidence_level": 0.95, "mean": 0.20683834036783905, "replicates": 8, "sample_sd": 0.00967059727409852, "standard_error": 0.0034190724553196023 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06019971817452064, 0.06378417510912215 ], "confidence_level": 0.95, "mean": 0.062001629467115696, "replicates": 8, "sample_sd": 0.0027674912768509553, "standard_error": 0.0009784559243679637 }, "log_loss": { "bootstrap_interval": [ 0.1998672043594227, 0.21184176489524092 ], "confidence_level": 0.95, "mean": 0.20602773709748298, "replicates": 8, "sample_sd": 0.009272956407232679, "standard_error": 0.0032784851786007356 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.9090909090909091 ], "oracle_method": "oracle", "population_optimal_scalar": 0.9090909090909091, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.25-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.008056384872921433, -0.006519797965590685 ], "confidence_level": 0.95, "mean": -0.007256802305165021, "replicates": 8, "sample_sd": 0.0012064681717904834, "standard_error": 0.0004265509127793936 }, "log_loss": { "bootstrap_interval": [ -0.024341795940723127, -0.019309393067732786 ], "confidence_level": 0.95, "mean": -0.0217810615755272, "replicates": 8, "sample_sd": 0.0038694284417717475, "standard_error": 0.0013680495452464492 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0012669283656087807, -0.0004703747316666604 ], "confidence_level": 0.95, "mean": -0.0008699780107401835, "replicates": 8, "sample_sd": 0.0006079641793702848, "standard_error": 0.00021494779697562143 }, "log_loss": { "bootstrap_interval": [ -0.0050936179993517575, -0.0010006967435801906 ], "confidence_level": 0.95, "mean": -0.003257915398908559, "replicates": 8, "sample_sd": 0.0031393748430216975, "standard_error": 0.0011099366200935476 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0002608267189526045, 0.0011031722538488165 ], "confidence_level": 0.95, "mean": 0.0006484427054523206, "replicates": 8, "sample_sd": 0.0006546346589253965, "standard_error": 0.00023144830326294526 }, "log_loss": { "bootstrap_interval": [ 0.0007888150338790208, 0.004588077735562593 ], "confidence_level": 0.95, "mean": 0.0025534615295308843, "replicates": 8, "sample_sd": 0.0029060760658122364, "standard_error": 0.0010274530463898778 } } }, "discount": { "bootstrap_interval": [ 0.7207857318475157, 0.8788293663781288 ], "confidence_level": 0.95, "mean": 0.8007339961480096, "replicates": 8, "sample_sd": 0.1222564281089059, "standard_error": 0.043224174679726494 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.9800618103419994, "discount_min": 0.6257097871058834, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18180935717073254, 0.18394981532381552 ], "confidence_level": 0.95, "mean": 0.18285178662062038, "replicates": 8, "sample_sd": 0.0016501441560566518, "standard_error": 0.0005834140613415055 }, "log_loss": { "bootstrap_interval": [ 0.5445335582143478, 0.5492374755378073 ], "confidence_level": 0.95, "mean": 0.5468518247646136, "replicates": 8, "sample_sd": 0.00367465235047127, "standard_error": 0.0012991857977606603 } }, "independent": { "brier": { "bootstrap_interval": [ 0.17461059621928784, 0.17833699682111595 ], "confidence_level": 0.95, "mean": 0.17646496232619555, "replicates": 8, "sample_sd": 0.0029162612305782105, "standard_error": 0.0010310540459266392 }, "log_loss": { "bootstrap_interval": [ 0.5234545667655482, 0.5332363049133765 ], "confidence_level": 0.95, "mean": 0.5283286785879949, "replicates": 8, "sample_sd": 0.0076938595279373335, "standard_error": 0.002720190122850609 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19672552101053392, 0.20066872593205412 ], "confidence_level": 0.95, "mean": 0.19864580941794724, "replicates": 8, "sample_sd": 0.0030921164901485903, "standard_error": 0.0010932282692014072 }, "log_loss": { "bootstrap_interval": [ 0.5751087239057384, 0.5845439826396218 ], "confidence_level": 0.95, "mean": 0.579797825332825, "replicates": 8, "sample_sd": 0.007299326707341479, "standard_error": 0.0025807017064286165 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17388445162486543, 0.17733268967539056 ], "confidence_level": 0.95, "mean": 0.17559498431545537, "replicates": 8, "sample_sd": 0.002649471408554889, "standard_error": 0.0009367295997745179 }, "log_loss": { "bootstrap_interval": [ 0.5207990581148197, 0.529453149856841 ], "confidence_level": 0.95, "mean": 0.5250707631890864, "replicates": 8, "sample_sd": 0.006671270959097505, "standard_error": 0.002358650467155364 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1733811318537572, 0.17658019648496526 ], "confidence_level": 0.95, "mean": 0.17494654161000303, "replicates": 8, "sample_sd": 0.002464295127655271, "standard_error": 0.0008712598978050053 }, "log_loss": { "bootstrap_interval": [ 0.5185722808834057, 0.5264587821097432 ], "confidence_level": 0.95, "mean": 0.5225173016595555, "replicates": 8, "sample_sd": 0.006116206581149067, "standard_error": 0.0021624055743341476 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.8 ], "oracle_method": "oracle", "population_optimal_scalar": 0.8, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.25-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.014486447560473955, -0.011849474389808077 ], "confidence_level": 0.95, "mean": -0.013178888258687658, "replicates": 8, "sample_sd": 0.00200766056673156, "standard_error": 0.0007098152005283566 }, "log_loss": { "bootstrap_interval": [ -0.058518129041513225, -0.048873648895357225 ], "confidence_level": 0.95, "mean": -0.05373122426637143, "replicates": 8, "sample_sd": 0.007538541664171015, "standard_error": 0.002665276965496323 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0007094889002161799, -0.0003216082949139651 ], "confidence_level": 0.95, "mean": -0.0004989804647253272, "replicates": 8, "sample_sd": 0.00030385958921518084, "standard_error": 0.00010743058803130654 }, "log_loss": { "bootstrap_interval": [ -0.005604179228857271, -0.002021493129254464 ], "confidence_level": 0.95, "mean": -0.0038745872628708593, "replicates": 8, "sample_sd": 0.002806965634695707, "standard_error": 0.0009924122174254678 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.9639946982756974e-05, 0.0006265624235179974 ], "confidence_level": 0.95, "mean": 0.0002714489131528299, "replicates": 8, "sample_sd": 0.0004798035003375462, "standard_error": 0.0001696361543628604 }, "log_loss": { "bootstrap_interval": [ 0.0002541143958371024, 0.0036301180977286935 ], "confidence_level": 0.95, "mean": 0.0017582243246642773, "replicates": 8, "sample_sd": 0.002667842762507243, "standard_error": 0.0009432248542541617 } } }, "discount": { "bootstrap_interval": [ 0.760920396167101, 0.8846219848213754 ], "confidence_level": 0.95, "mean": 0.8253331322731678, "replicates": 8, "sample_sd": 0.09653978169522404, "standard_error": 0.034131967145480925 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.971705276893069, "discount_min": 0.6653022305628831, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08654384842625343, 0.08927322772740727 ], "confidence_level": 0.95, "mean": 0.08788736217460304, "replicates": 8, "sample_sd": 0.0021346984358096757, "standard_error": 0.0007547298698746687 }, "log_loss": { "bootstrap_interval": [ 0.2957015526864146, 0.3022586781553838 ], "confidence_level": 0.95, "mean": 0.29894388523682747, "replicates": 8, "sample_sd": 0.005126754482468029, "standard_error": 0.001812581430015836 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07288933509890919, 0.07759043341202511 ], "confidence_level": 0.95, "mean": 0.0752074543806407, "replicates": 8, "sample_sd": 0.003686264827551566, "standard_error": 0.0013032914284055858 }, "log_loss": { "bootstrap_interval": [ 0.24082861583116993, 0.2568559764528756 ], "confidence_level": 0.95, "mean": 0.24908724823332692, "replicates": 8, "sample_sd": 0.012579942714279225, "standard_error": 0.004447681400102571 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11037906427742983, 0.11427200956272957 ], "confidence_level": 0.95, "mean": 0.11236518873898231, "replicates": 8, "sample_sd": 0.003048391672834064, "standard_error": 0.001077769211786785 }, "log_loss": { "bootstrap_interval": [ 0.34790939799772674, 0.359123216537475 ], "confidence_level": 0.95, "mean": 0.3538358241699904, "replicates": 8, "sample_sd": 0.00876608933492542, "standard_error": 0.003099280606606418 } }, "learned": { "brier": { "bootstrap_interval": [ 0.07244362711133862, 0.0770461967749275 ], "confidence_level": 0.95, "mean": 0.07470847391591538, "replicates": 8, "sample_sd": 0.003574833861513472, "standard_error": 0.0012638946325457337 }, "log_loss": { "bootstrap_interval": [ 0.23787793079561806, 0.25274656783597077 ], "confidence_level": 0.95, "mean": 0.24521266097045605, "replicates": 8, "sample_sd": 0.011339721442296107, "standard_error": 0.004009196964307037 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.07233917267124827, 0.07658087587371099 ], "confidence_level": 0.95, "mean": 0.07443702500276254, "replicates": 8, "sample_sd": 0.00321555317158788, "standard_error": 0.00113686972644785 }, "log_loss": { "bootstrap_interval": [ 0.2371041335426841, 0.24957290905739754 ], "confidence_level": 0.95, "mean": 0.24345443664579178, "replicates": 8, "sample_sd": 0.0096925371597622, "standard_error": 0.003426829376285225 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.8 ], "oracle_method": "oracle", "population_optimal_scalar": 0.8, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.5-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0028179659424898166, -0.0014240131572632012 ], "confidence_level": 0.95, "mean": -0.0021745994858951394, "replicates": 8, "sample_sd": 0.0010764384908965227, "standard_error": 0.00038057847822157244 }, "log_loss": { "bootstrap_interval": [ -0.008633156589242977, -0.0033125740092270067 ], "confidence_level": 0.95, "mean": -0.006287054639267486, "replicates": 8, "sample_sd": 0.004215950874232257, "standard_error": 0.0014905637261594911 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.005016456737448408, -0.0038198455798761494 ], "confidence_level": 0.95, "mean": -0.004438052341089498, "replicates": 8, "sample_sd": 0.0009286666071201885, "standard_error": 0.0003283332276780943 }, "log_loss": { "bootstrap_interval": [ -0.023502893776787718, -0.017092862764824412 ], "confidence_level": 0.95, "mean": -0.02021798892642218, "replicates": 8, "sample_sd": 0.004898816807669171, "standard_error": 0.0017319932922467529 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.123891860527949e-05, 0.0012543032253379816 ], "confidence_level": 0.95, "mean": 0.0006015134785638844, "replicates": 8, "sample_sd": 0.0009063931967761858, "standard_error": 0.0003204583879308968 }, "log_loss": { "bootstrap_interval": [ 0.0002053698249857433, 0.0048257237162784485 ], "confidence_level": 0.95, "mean": 0.002260203401180688, "replicates": 8, "sample_sd": 0.0036535971039151033, "standard_error": 0.0012917416439509502 } } }, "discount": { "bootstrap_interval": [ 0.6053283496643328, 0.7336864048586363 ], "confidence_level": 0.95, "mean": 0.6695656850380121, "replicates": 8, "sample_sd": 0.09990420390787844, "standard_error": 0.035321470026152206 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8349044089669275, "discount_min": 0.5084273321931103, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18766800553848084, 0.18999493671392725 ], "confidence_level": 0.95, "mean": 0.18876323135035034, "replicates": 8, "sample_sd": 0.0018261852028537584, "standard_error": 0.0006456539703202117 }, "log_loss": { "bootstrap_interval": [ 0.5563883843961787, 0.5619680621074072 ], "confidence_level": 0.95, "mean": 0.5590141383214298, "replicates": 8, "sample_sd": 0.004357964752403109, "standard_error": 0.001540773214298096 } }, "independent": { "brier": { "bootstrap_interval": [ 0.18924292053753738, 0.19300055310786962 ], "confidence_level": 0.95, "mean": 0.1910266842055447, "replicates": 8, "sample_sd": 0.0029139323928333025, "standard_error": 0.0010302306774457853 }, "log_loss": { "bootstrap_interval": [ 0.5671761100429895, 0.5796863309074579 ], "confidence_level": 0.95, "mean": 0.5729450726085845, "replicates": 8, "sample_sd": 0.009748493416162506, "standard_error": 0.00344661290046046 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19771582525386094, 0.20077084867065273 ], "confidence_level": 0.95, "mean": 0.19926702311469635, "replicates": 8, "sample_sd": 0.0023311563873632864, "standard_error": 0.0008241882447554569 }, "log_loss": { "bootstrap_interval": [ 0.5780452812068545, 0.585061234082303 ], "confidence_level": 0.95, "mean": 0.581755125408795, "replicates": 8, "sample_sd": 0.005465067828045487, "standard_error": 0.0019321932604277003 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18511232901377137, 0.18825849434609154 ], "confidence_level": 0.95, "mean": 0.1865886318644552, "replicates": 8, "sample_sd": 0.0024208632531367035, "standard_error": 0.0008559044113091443 }, "log_loss": { "bootstrap_interval": [ 0.5486722309584391, 0.5575456924810777 ], "confidence_level": 0.95, "mean": 0.5527270836821623, "replicates": 8, "sample_sd": 0.006878257191744205, "standard_error": 0.002431831151513733 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18457663555850368, 0.18754959207274327 ], "confidence_level": 0.95, "mean": 0.18598711838589133, "replicates": 8, "sample_sd": 0.002283938612703542, "standard_error": 0.0008074942404282351 }, "log_loss": { "bootstrap_interval": [ 0.5467853459680957, 0.5547922282324368 ], "confidence_level": 0.95, "mean": 0.5504668802809816, "replicates": 8, "sample_sd": 0.006134282591435246, "standard_error": 0.002168796409059225 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.6666666666666666 ], "oracle_method": "oracle", "population_optimal_scalar": 0.6666666666666666, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.5-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.006310981409210276, -0.0038739802205223217 ], "confidence_level": 0.95, "mean": -0.005188152361846015, "replicates": 8, "sample_sd": 0.0019030902864656443, "standard_error": 0.0006728440233850531 }, "log_loss": { "bootstrap_interval": [ -0.025634251190482925, -0.014320051546877474 ], "confidence_level": 0.95, "mean": -0.020486027520469047, "replicates": 8, "sample_sd": 0.008672435686085887, "standard_error": 0.0030661690415177694 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.003419586197664597, -0.002272284343745147 ], "confidence_level": 0.95, "mean": -0.0027805913223125523, "replicates": 8, "sample_sd": 0.0008965003325825065, "standard_error": 0.00031696073225254275 }, "log_loss": { "bootstrap_interval": [ -0.028146450703791074, -0.019380335787584135 ], "confidence_level": 0.95, "mean": -0.023180437435310473, "replicates": 8, "sample_sd": 0.006743779983252572, "standard_error": 0.0023842862784939976 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 3.545345218012273e-05, 0.0009118996466519601 ], "confidence_level": 0.95, "mean": 0.00037732484976664427, "replicates": 8, "sample_sd": 0.0007285342889546153, "standard_error": 0.00025757576802336405 }, "log_loss": { "bootstrap_interval": [ 0.0004910870317722457, 0.00581992764525506 ], "confidence_level": 0.95, "mean": 0.0025843324381175856, "replicates": 8, "sample_sd": 0.004395507560587939, "standard_error": 0.0015540466014242354 } } }, "discount": { "bootstrap_interval": [ 0.6339114812905031, 0.750918386034936 ], "confidence_level": 0.95, "mean": 0.6934054673012884, "replicates": 8, "sample_sd": 0.09188056203813767, "standard_error": 0.03248468423819921 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8353465165709957, "discount_min": 0.5482500751005486, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09509396124301753, 0.09857066499633346 ], "confidence_level": 0.95, "mean": 0.09670426406137236, "replicates": 8, "sample_sd": 0.002719005553236389, "standard_error": 0.0009613136323886654 }, "log_loss": { "bootstrap_interval": [ 0.3130600374933035, 0.3222823804561464 ], "confidence_level": 0.95, "mean": 0.31739790841878673, "replicates": 8, "sample_sd": 0.007124592990997274, "standard_error": 0.0025189240085641594 } }, "independent": { "brier": { "bootstrap_interval": [ 0.09148215292362334, 0.09743520665299049 ], "confidence_level": 0.95, "mean": 0.0942967030218389, "replicates": 8, "sample_sd": 0.004671234420607927, "standard_error": 0.001651530767661939 }, "log_loss": { "bootstrap_interval": [ 0.30850664813233236, 0.3327852737927832 ], "confidence_level": 0.95, "mean": 0.32009231833362817, "replicates": 8, "sample_sd": 0.018801559443550074, "standard_error": 0.006647355089708114 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11190974686860967, 0.11541868657444712 ], "confidence_level": 0.95, "mean": 0.11374706502855107, "replicates": 8, "sample_sd": 0.0026832453909882235, "standard_error": 0.0009486705057776608 }, "log_loss": { "bootstrap_interval": [ 0.35296883297717163, 0.36238237269542917 ], "confidence_level": 0.95, "mean": 0.3585218932160255, "replicates": 8, "sample_sd": 0.007593125351894876, "standard_error": 0.0026845752133621784 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08909007761672297, 0.09435281637834597 ], "confidence_level": 0.95, "mean": 0.09151611169952634, "replicates": 8, "sample_sd": 0.004044509739794755, "standard_error": 0.0014299501317919549 }, "log_loss": { "bootstrap_interval": [ 0.2886242719829753, 0.306495550401719 ], "confidence_level": 0.95, "mean": 0.29691188089831766, "replicates": 8, "sample_sd": 0.013805939713806578, "standard_error": 0.004881136796142647 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08891428375552635, 0.09361863262586902 ], "confidence_level": 0.95, "mean": 0.0911387868497597, "replicates": 8, "sample_sd": 0.0037049044242628802, "standard_error": 0.001309881521022162 }, "log_loss": { "bootstrap_interval": [ 0.2874249139812205, 0.3020889000437128 ], "confidence_level": 0.95, "mean": 0.2943275484602001, "replicates": 8, "sample_sd": 0.011457885286723772, "standard_error": 0.0040509741921499745 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.6666666666666666 ], "oracle_method": "oracle", "population_optimal_scalar": 0.6666666666666666, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.75-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0005078121519911487, 0.0009500447711146866 ], "confidence_level": 0.95, "mean": 0.00013308313992233672, "replicates": 8, "sample_sd": 0.001166914279617261, "standard_error": 0.0004125665000903901 }, "log_loss": { "bootstrap_interval": [ -0.0015763519548111088, 0.004402132283029662 ], "confidence_level": 0.95, "mean": 0.0009142476810472672, "replicates": 8, "sample_sd": 0.004873080896835566, "standard_error": 0.0017228942737115257 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.009531876471762757, -0.007788752453979808 ], "confidence_level": 0.95, "mean": -0.008684297485067646, "replicates": 8, "sample_sd": 0.0013628266899089457, "standard_error": 0.0004818319970083158 }, "log_loss": { "bootstrap_interval": [ -0.04719229039263619, -0.0381827159446132 ], "confidence_level": 0.95, "mean": -0.04218965376299176, "replicates": 8, "sample_sd": 0.006949648558991133, "standard_error": 0.002457071811462974 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -4.523708473584687e-06, 0.0013867625248950993 ], "confidence_level": 0.95, "mean": 0.0006028567599527176, "replicates": 8, "sample_sd": 0.0011054874890562966, "standard_error": 0.00039084885001429827 }, "log_loss": { "bootstrap_interval": [ -0.00011155037141194856, 0.005393909462392238 ], "confidence_level": 0.95, "mean": 0.0022612901650592293, "replicates": 8, "sample_sd": 0.004490198431580402, "standard_error": 0.0015875248799218512 } } }, "discount": { "bootstrap_interval": [ 0.5219588771370431, 0.636814436386964 ], "confidence_level": 0.95, "mean": 0.5786366968324972, "replicates": 8, "sample_sd": 0.0873872103130906, "standard_error": 0.030896044500680682 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.7406629951735497, "discount_min": 0.43088527542078126, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19332194996789714, 0.196093325761138 ], "confidence_level": 0.95, "mean": 0.19461482539958203, "replicates": 8, "sample_sd": 0.002130848550905638, "standard_error": 0.0007533687300134523 }, "log_loss": { "bootstrap_interval": [ 0.5687088779635654, 0.5758435651350275 ], "confidence_level": 0.95, "mean": 0.5719844366758325, "replicates": 8, "sample_sd": 0.005519201941186299, "standard_error": 0.001951332559675394 } }, "independent": { "brier": { "bootstrap_interval": [ 0.20145043241447033, 0.2055370117105403 ], "confidence_level": 0.95, "mean": 0.20343220602457202, "replicates": 8, "sample_sd": 0.0031828656924365877, "standard_error": 0.0011253129573639636 }, "log_loss": { "bootstrap_interval": [ 0.6076775556056438, 0.624003042692003 ], "confidence_level": 0.95, "mean": 0.6150883381198716, "replicates": 8, "sample_sd": 0.01269139345312116, "standard_error": 0.004487085186704263 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19856338917473088, 0.20112537074412104 ], "confidence_level": 0.95, "mean": 0.19980328185793006, "replicates": 8, "sample_sd": 0.001967302994709199, "standard_error": 0.0006955466441037386 }, "log_loss": { "bootstrap_interval": [ 0.5806655423721475, 0.5867845780718124 ], "confidence_level": 0.95, "mean": 0.5837083029857186, "replicates": 8, "sample_sd": 0.004731593149177854, "standard_error": 0.0016728708007997361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19320589345548234, 0.19651718398424653 ], "confidence_level": 0.95, "mean": 0.19474790853950436, "replicates": 8, "sample_sd": 0.002525041987920764, "standard_error": 0.0008927371562197662 }, "log_loss": { "bootstrap_interval": [ 0.5682856173242812, 0.578372477794567 ], "confidence_level": 0.95, "mean": 0.5728986843568799, "replicates": 8, "sample_sd": 0.00792573907338803, "standard_error": 0.0028021719223539293 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19268032514319314, 0.19574081241496175 ], "confidence_level": 0.95, "mean": 0.19414505177955166, "replicates": 8, "sample_sd": 0.0023728313085090587, "standard_error": 0.000838922554429252 }, "log_loss": { "bootstrap_interval": [ 0.5667594632011549, 0.5751296779293006 ], "confidence_level": 0.95, "mean": 0.5706373941918206, "replicates": 8, "sample_sd": 0.006572343525319504, "standard_error": 0.0023236743375204603 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5714285714285714 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5714285714285714, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho0.75-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.001662132527256882, 0.00014400106132685643 ], "confidence_level": 0.95, "mean": -0.0008966617112012628, "replicates": 8, "sample_sd": 0.0014148497631481139, "standard_error": 0.000500224930941106 }, "log_loss": { "bootstrap_interval": [ -0.006186269135857993, 0.005174732995516712 ], "confidence_level": 0.95, "mean": -0.0016795562100352657, "replicates": 8, "sample_sd": 0.009289400363536501, "standard_error": 0.0032842989951067194 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0063289832439712935, -0.004503586721159298 ], "confidence_level": 0.95, "mean": -0.005252599306984032, "replicates": 8, "sample_sd": 0.0014726855138163302, "standard_error": 0.0005206729566873611 }, "log_loss": { "bootstrap_interval": [ -0.06036109842028179, -0.04489050427563113 ], "confidence_level": 0.95, "mean": -0.05154090872896415, "replicates": 8, "sample_sd": 0.012213462163262054, "standard_error": 0.004318110958703959 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -3.926154681616894e-05, 0.0011401838569181783 ], "confidence_level": 0.95, "mean": 0.0004440504234385339, "replicates": 8, "sample_sd": 0.0009430282016382236, "standard_error": 0.0003334108181142714 }, "log_loss": { "bootstrap_interval": [ -0.00016428302142354942, 0.008596348119029797 ], "confidence_level": 0.95, "mean": 0.003241536705614549, "replicates": 8, "sample_sd": 0.007273374884033414, "standard_error": 0.002571526351305973 } } }, "discount": { "bootstrap_interval": [ 0.5371518184382009, 0.6521615782422638 ], "confidence_level": 0.95, "mean": 0.5920070995091968, "replicates": 8, "sample_sd": 0.09193004013911839, "standard_error": 0.03250217738856106 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.7567299066330309, "discount_min": 0.4789756502639315, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.10394419754105318, 0.10811888066521386 ], "confidence_level": 0.95, "mean": 0.10579064999555343, "replicates": 8, "sample_sd": 0.0032532378256890085, "standard_error": 0.0011501932636786387 }, "log_loss": { "bootstrap_interval": [ 0.33386897964896256, 0.3461102960899949 ], "confidence_level": 0.95, "mean": 0.3395720016766682, "replicates": 8, "sample_sd": 0.009622715792010458, "standard_error": 0.003402143794980737 } }, "independent": { "brier": { "bootstrap_interval": [ 0.10713531908314906, 0.11380492738818425 ], "confidence_level": 0.95, "mean": 0.1101465875913362, "replicates": 8, "sample_sd": 0.005207809942353258, "standard_error": 0.0018412388626843558 }, "log_loss": { "bootstrap_interval": [ 0.37477286062608584, 0.4065444488939611 ], "confidence_level": 0.95, "mean": 0.3894333541955971, "replicates": 8, "sample_sd": 0.024615286537302924, "standard_error": 0.008702818015688413 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11268326276074059, 0.11678374892552676 ], "confidence_level": 0.95, "mean": 0.11469046688997823, "replicates": 8, "sample_sd": 0.003156091016768506, "standard_error": 0.0011158466799994781 }, "log_loss": { "bootstrap_interval": [ 0.35680589389965756, 0.3686357975871915 ], "confidence_level": 0.95, "mean": 0.3628740273563677, "replicates": 8, "sample_sd": 0.008938420049611378, "standard_error": 0.0031602087150870008 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10242914000802866, 0.10794772908254346 ], "confidence_level": 0.95, "mean": 0.10489398828435217, "replicates": 8, "sample_sd": 0.004136661177100052, "standard_error": 0.0014625305848992862 }, "log_loss": { "bootstrap_interval": [ 0.3286124602313119, 0.3494691764298849 ], "confidence_level": 0.95, "mean": 0.33789244546663294, "replicates": 8, "sample_sd": 0.016088109475702623, "standard_error": 0.005688005653370438 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10221613846401706, 0.10717691137302189 ], "confidence_level": 0.95, "mean": 0.10444993786091364, "replicates": 8, "sample_sd": 0.00389444235977127, "standard_error": 0.0013768933007672025 }, "log_loss": { "bootstrap_interval": [ 0.32716689088301115, 0.3434218254349929 ], "confidence_level": 0.95, "mean": 0.3346509087610184, "replicates": 8, "sample_sd": 0.012580572243538252, "standard_error": 0.004447903972306578 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5714285714285714 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5714285714285714, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho1-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 8.958643619100037e-05, 0.0015875298228479869 ], "confidence_level": 0.95, "mean": 0.0007045402269900518, "replicates": 8, "sample_sd": 0.001222778152552083, "standard_error": 0.0004323173617781683 }, "log_loss": { "bootstrap_interval": [ 0.00029883563667936554, 0.0067968036203644305 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.014187958084807648, -0.012102286163708573 ], "confidence_level": 0.95, "mean": -0.013131091973749981, "replicates": 8, "sample_sd": 0.001644178922080863, "standard_error": 0.0005813050326436832 }, "log_loss": { "bootstrap_interval": [ -0.07183510466372435, -0.06214859666620257 ], "confidence_level": 0.95, "mean": -0.06643195649026201, "replicates": 8, "sample_sd": 0.007532633277318459, "standard_error": 0.002663188035291665 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.958643619100037e-05, 0.0015875298228479869 ], "confidence_level": 0.95, "mean": 0.0007045402269900518, "replicates": 8, "sample_sd": 0.001222778152552083, "standard_error": 0.0004323173617781683 }, "log_loss": { "bootstrap_interval": [ 0.00029883563667936554, 0.0067968036203644305 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } } }, "discount": { "bootstrap_interval": [ 0.4634659347599442, 0.571907950220405 ], "confidence_level": 0.95, "mean": 0.5148797067418193, "replicates": 8, "sample_sd": 0.08329514899395475, "standard_error": 0.029449282346784615 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.676943818372854, "discount_min": 0.39065764671069125, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19898261686923327, 0.20254050556348285 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5821706246995105, 0.5914392304014728 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "independent": { "brier": { "bootstrap_interval": [ 0.21188916918683257, 0.2170925350342194 ], "confidence_level": 0.95, "mean": 0.2145667743911054, "replicates": 8, "sample_sd": 0.004023456566401655, "standard_error": 0.0014225067109560762 }, "log_loss": { "bootstrap_interval": [ 0.6464356038670201, 0.6661524998536238 ], "confidence_level": 0.95, "mean": 0.6557975952040205, "replicates": 8, "sample_sd": 0.015471081104813612, "standard_error": 0.005469853180750384 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19900554837325526, 0.20253460845817206 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5822560315782703, 0.5914530370156917 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1993983638289941, 0.2034488635789732 ], "confidence_level": 0.95, "mean": 0.20143568241735543, "replicates": 8, "sample_sd": 0.00314778408765836, "standard_error": 0.001112909737047168 }, "log_loss": { "bootstrap_interval": [ 0.583513673697855, 0.596147058564755 ], "confidence_level": 0.95, "mean": 0.5893656387137585, "replicates": 8, "sample_sd": 0.009854990310042909, "standard_error": 0.0034842652383795286 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19898257741413214, 0.20254050216783043 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5821980692227148, 0.5914737593539241 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n2-rho1-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 8.344488287256647e-05, 0.001597508888844728 ], "confidence_level": 0.95, "mean": 0.0007233667431064544, "replicates": 8, "sample_sd": 0.0011947528903027151, "standard_error": 0.00042240893528763855 }, "log_loss": { "bootstrap_interval": [ 0.00011414315819316401, 0.01620336961290359 ], "confidence_level": 0.95, "mean": 0.006085127988558439, "replicates": 8, "sample_sd": 0.013870011259328728, "standard_error": 0.004903789508302554 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.009660839806475804, -0.006699604031904768 ], "confidence_level": 0.95, "mean": -0.008160751179664603, "replicates": 8, "sample_sd": 0.0022697859382974014, "standard_error": 0.0008024905144059815 }, "log_loss": { "bootstrap_interval": [ -0.0941670119616759, -0.07509060452193063 ], "confidence_level": 0.95, "mean": -0.08352156077746609, "replicates": 8, "sample_sd": 0.014735731789591897, "standard_error": 0.005209867937083305 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.344488287256647e-05, 0.001597508888844728 ], "confidence_level": 0.95, "mean": 0.0007233667431064544, "replicates": 8, "sample_sd": 0.0011947528903027151, "standard_error": 0.00042240893528763855 }, "log_loss": { "bootstrap_interval": [ 0.00011414315819317095, 0.01620336961290359 ], "confidence_level": 0.95, "mean": 0.006085127988558446, "replicates": 8, "sample_sd": 0.013870011259328725, "standard_error": 0.004903789508302553 } } }, "discount": { "bootstrap_interval": [ 0.4585196351989872, 0.5983347886941739 ], "confidence_level": 0.95, "mean": 0.5201354278247174, "replicates": 8, "sample_sd": 0.10885528250855786, "standard_error": 0.038486154214889316 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.745459360493919, "discount_min": 0.42852866135966206, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.11369043270453144, 0.11834584536787959 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.3602772618304846, 0.37621159121438136 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "independent": { "brier": { "bootstrap_interval": [ 0.12180217389038847, 0.12819875050400395 ], "confidence_level": 0.95, "mean": 0.12470676554916196, "replicates": 8, "sample_sd": 0.004950968958328187, "standard_error": 0.001750431861938979 }, "log_loss": { "bootstrap_interval": [ 0.4413301823128574, 0.4756969912101825 ], "confidence_level": 0.95, "mean": 0.4573494224976451, "replicates": 8, "sample_sd": 0.02658573114483914, "standard_error": 0.009399475387659074 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1136754964477846, 0.11832291075374618 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.36031216823149453, 0.3762270281379732 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11398388827690414, 0.11932141904079258 ], "confidence_level": 0.95, "mean": 0.11654601436949735, "replicates": 8, "sample_sd": 0.004073392697855984, "standard_error": 0.0014401617995448658 }, "log_loss": { "bootstrap_interval": [ 0.36213842152023784, 0.38990309837721365 ], "confidence_level": 0.95, "mean": 0.373827861720179, "replicates": 8, "sample_sd": 0.022116164976176092, "standard_error": 0.007819245114247267 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11368241336453012, 0.11834506339812631 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.36041502070155007, 0.3761670454805101 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.15324316448776326, -0.15002594532590166 ], "confidence_level": 0.95, "mean": -0.15163635034228307, "replicates": 8, "sample_sd": 0.0025074069353770547, "standard_error": 0.0008865022235996473 }, "log_loss": { "bootstrap_interval": [ -0.48300142624000025, -0.4725075638231767 ], "confidence_level": 0.95, "mean": -0.4777371006716403, "replicates": 8, "sample_sd": 0.00807636304730641, "standard_error": 0.002855425539037406 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -8.195186629783905e-06, 0.00011020060614887265 ], "confidence_level": 0.95, "mean": 4.146390659584586e-05, "replicates": 8, "sample_sd": 9.396607524696785e-05, "standard_error": 3.322202450430818e-05 }, "log_loss": { "bootstrap_interval": [ 0.0002870250743140213, 0.0013046634275670006 ], "confidence_level": 0.95, "mean": 0.000795844250940511, "replicates": 8, "sample_sd": 0.0007725130339819302, "standard_error": 0.0002731246024418083 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -8.195186629783905e-06, 0.00011020060614887265 ], "confidence_level": 0.95, "mean": 4.146390659584586e-05, "replicates": 8, "sample_sd": 9.396607524696785e-05, "standard_error": 3.322202450430818e-05 }, "log_loss": { "bootstrap_interval": [ 0.0002870250743140213, 0.0013046634275670006 ], "confidence_level": 0.95, "mean": 0.000795844250940511, "replicates": 8, "sample_sd": 0.0007725130339819302, "standard_error": 0.0002731246024418083 } } }, "discount": { "bootstrap_interval": [ 0.8204373640041475, 0.9581534680084539 ], "confidence_level": 0.95, "mean": 0.8898437667366547, "replicates": 8, "sample_sd": 0.10719197429548005, "standard_error": 0.037898085956554015 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7104358182782733, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16073035662031399, 0.1613018240925052 ], "confidence_level": 0.95, "mean": 0.16102634253225062, "replicates": 8, "sample_sd": 0.00043761098191535274, "standard_error": 0.00015471884641702477 }, "log_loss": { "bootstrap_interval": [ 0.5109588422831599, 0.5121483949292103 ], "confidence_level": 0.95, "mean": 0.5115876095720275, "replicates": 8, "sample_sd": 0.000908876722534737, "standard_error": 0.0003213364468834583 } }, "independent": { "brier": { "bootstrap_interval": [ 0.007918621153390728, 0.010788126744907284 ], "confidence_level": 0.95, "mean": 0.009348528283371712, "replicates": 8, "sample_sd": 0.0022396895709082313, "standard_error": 0.0007918498416709996 }, "log_loss": { "bootstrap_interval": [ 0.028172481412086094, 0.03791088600281982 ], "confidence_level": 0.95, "mean": 0.033054664649446716, "replicates": 8, "sample_sd": 0.007624976541644333, "standard_error": 0.0026958363094925285 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1946761340940113, 0.2009843031942414 ], "confidence_level": 0.95, "mean": 0.19761153359360628, "replicates": 8, "sample_sd": 0.004840249158008265, "standard_error": 0.0017112865011300604 }, "log_loss": { "bootstrap_interval": [ 0.5704888265160561, 0.5846511146906339 ], "confidence_level": 0.95, "mean": 0.5772891634083406, "replicates": 8, "sample_sd": 0.01097636370028827, "standard_error": 0.0038807306026218498 } }, "learned": { "brier": { "bootstrap_interval": [ 0.007987475870297238, 0.010833532861879682 ], "confidence_level": 0.95, "mean": 0.009389992189967559, "replicates": 8, "sample_sd": 0.0021892176025017276, "standard_error": 0.0007740053061109635 }, "log_loss": { "bootstrap_interval": [ 0.029027108077766987, 0.03866559740648011 ], "confidence_level": 0.95, "mean": 0.033850508900387224, "replicates": 8, "sample_sd": 0.007452231064737551, "standard_error": 0.002634761560422484 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.007911030947267448, 0.010793604532880986 ], "confidence_level": 0.95, "mean": 0.009348528283371712, "replicates": 8, "sample_sd": 0.0022396895709082313, "standard_error": 0.0007918498416709996 }, "log_loss": { "bootstrap_interval": [ 0.02810545699829361, 0.03799966357547015 ], "confidence_level": 0.95, "mean": 0.033054664649446716, "replicates": 8, "sample_sd": 0.007624976541644333, "standard_error": 0.0026958363094925285 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05414945164372605, -0.053823547061859475 ], "confidence_level": 0.95, "mean": -0.054005506274376114, "replicates": 8, "sample_sd": 0.0002542190247302897, "standard_error": 8.987999814670922e-05 }, "log_loss": { "bootstrap_interval": [ -0.2588609221005691, -0.2578584092814701 ], "confidence_level": 0.95, "mean": -0.2584144389682944, "replicates": 8, "sample_sd": 0.0007769692215447824, "standard_error": 0.0002747001026637743 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "discount": { "bootstrap_interval": [ 1.0, 1.0 ], "confidence_level": 0.95, "mean": 1.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 1.0, "discount_one_count": 8, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05382588691982943, 0.05415198099320278 ], "confidence_level": 0.95, "mean": 0.0540055220645327, "replicates": 8, "sample_sd": 0.00025424010692824136, "standard_error": 8.98874518292762e-05 }, "log_loss": { "bootstrap_interval": [ 0.25786540011307246, 0.2588706989384176 ], "confidence_level": 0.95, "mean": 0.2584158735863138, "replicates": 8, "sample_sd": 0.0007785353451742604, "standard_error": 0.0002752538109830645 } }, "independent": { "brier": { "bootstrap_interval": [ 1.2325545238223475e-13, 4.736774201254424e-08 ], "confidence_level": 0.95, "mean": 1.5790156584646467e-08, "replicates": 8, "sample_sd": 4.4656903382896313e-08, "standard_error": 1.5788599604419227e-08 }, "log_loss": { "bootstrap_interval": [ 7.5697907171146e-09, 4.259554195071665e-06 ], "confidence_level": 0.95, "mean": 1.434618019337677e-06, "replicates": 8, "sample_sd": 3.98781666803232e-06, "standard_error": 1.4099061040471983e-06 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10677960862488178, 0.11304618603367605 ], "confidence_level": 0.95, "mean": 0.10971962570460989, "replicates": 8, "sample_sd": 0.004901062078073617, "standard_error": 0.0017327871152110436 }, "log_loss": { "bootstrap_interval": [ 0.3399717401921756, 0.35593025909801834 ], "confidence_level": 0.95, "mean": 0.3478498156750285, "replicates": 8, "sample_sd": 0.01248682896093513, "standard_error": 0.0044147607168969006 } }, "learned": { "brier": { "bootstrap_interval": [ 1.187814964054086e-13, 4.7367746457842116e-08 ], "confidence_level": 0.95, "mean": 1.5790156584646467e-08, "replicates": 8, "sample_sd": 4.4656903382896313e-08, "standard_error": 1.5788599604419227e-08 }, "log_loss": { "bootstrap_interval": [ 7.548595646183092e-09, 4.259116837410866e-06 ], "confidence_level": 0.95, "mean": 1.434618019337677e-06, "replicates": 8, "sample_sd": 3.98781666803232e-06, "standard_error": 1.4099061040471983e-06 } }, "oracle": { "brier": { "bootstrap_interval": [ 1.2310747251256882e-13, 4.736774201254424e-08 ], "confidence_level": 0.95, "mean": 1.5790156584646467e-08, "replicates": 8, "sample_sd": 4.4656903382896313e-08, "standard_error": 1.5788599604419227e-08 }, "log_loss": { "bootstrap_interval": [ 7.548595646183092e-09, 4.25982982915889e-06 ], "confidence_level": 0.95, "mean": 1.434618019337677e-06, "replicates": 8, "sample_sd": 3.98781666803232e-06, "standard_error": 1.4099061040471983e-06 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.1-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.09350719350959276, -0.08970591446020271 ], "confidence_level": 0.95, "mean": -0.09145295890825697, "replicates": 8, "sample_sd": 0.002969532353370318, "standard_error": 0.0010498882320104993 }, "log_loss": { "bootstrap_interval": [ -0.283213643928428, -0.2698773214527895 ], "confidence_level": 0.95, "mean": -0.27672234143072066, "replicates": 8, "sample_sd": 0.010308645330766975, "standard_error": 0.0036446565091161837 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.01234987506549802, -0.00921498864894021 ], "confidence_level": 0.95, "mean": -0.010748642957558749, "replicates": 8, "sample_sd": 0.002451461350139887, "standard_error": 0.0008667224722503216 }, "log_loss": { "bootstrap_interval": [ -0.17812540246239672, -0.14856012907799954 ], "confidence_level": 0.95, "mean": -0.16232345854441602, "replicates": 8, "sample_sd": 0.023086564108461802, "standard_error": 0.00816233301769565 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00014128896307412893, 6.77315532764637e-05 ], "confidence_level": 0.95, "mean": -3.491861685914373e-05, "replicates": 8, "sample_sd": 0.0001600407412781471, "standard_error": 5.658294671194981e-05 }, "log_loss": { "bootstrap_interval": [ -2.6456058532973443e-05, 0.0013159498492623031 ], "confidence_level": 0.95, "mean": 0.0004913568602577904, "replicates": 8, "sample_sd": 0.0011309028173225415, "standard_error": 0.0003998345254958702 } } }, "discount": { "bootstrap_interval": [ 0.32548009828270646, 0.3556930904081148 ], "confidence_level": 0.95, "mean": 0.34014776928517315, "replicates": 8, "sample_sd": 0.023315629709950984, "standard_error": 0.008243319937770438 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.38327673660880723, "discount_min": 0.31036379521493285, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1646388873716603, 0.1658374286848839 ], "confidence_level": 0.95, "mean": 0.16517723867444667, "replicates": 8, "sample_sd": 0.0009388064666798529, "standard_error": 0.00033191820940555323 }, "log_loss": { "bootstrap_interval": [ 0.5165541028722463, 0.5191270470401432 ], "confidence_level": 0.95, "mean": 0.5177422842275725, "replicates": 8, "sample_sd": 0.002021899916609033, "standard_error": 0.0007148495709573811 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08136870863231846, 0.08776806924999207 ], "confidence_level": 0.95, "mean": 0.08447292272374844, "replicates": 8, "sample_sd": 0.0049539392470365345, "standard_error": 0.0017514820175828563 }, "log_loss": { "bootstrap_interval": [ 0.38296590269168856, 0.4276138790875029 ], "confidence_level": 0.95, "mean": 0.40334340134126784, "replicates": 8, "sample_sd": 0.0345265116451882, "standard_error": 0.012206965257514437 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19581417946903962, 0.20066776035542977 ], "confidence_level": 0.95, "mean": 0.1981676486512557, "replicates": 8, "sample_sd": 0.0038049404201851056, "standard_error": 0.0013452495865618396 }, "log_loss": { "bootstrap_interval": [ 0.5729021362609865, 0.5843364262248885 ], "confidence_level": 0.95, "mean": 0.578515600100781, "replicates": 8, "sample_sd": 0.008863064931613176, "standard_error": 0.00313356665762018 } }, "learned": { "brier": { "bootstrap_interval": [ 0.07156539578789589, 0.07597945252216527 ], "confidence_level": 0.95, "mean": 0.07372427976618968, "replicates": 8, "sample_sd": 0.003433822917085255, "standard_error": 0.0012140397350323777 }, "log_loss": { "bootstrap_interval": [ 0.23397821852560144, 0.24896551372154682 ], "confidence_level": 0.95, "mean": 0.2410199427968518, "replicates": 8, "sample_sd": 0.011524820079785107, "standard_error": 0.004074639215185468 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.07162698930779476, 0.0760185496005983 ], "confidence_level": 0.95, "mean": 0.07375919838304883, "replicates": 8, "sample_sd": 0.003420518837317243, "standard_error": 0.0012093360325216738 }, "log_loss": { "bootstrap_interval": [ 0.23382404933391168, 0.24827165537593338 ], "confidence_level": 0.95, "mean": 0.240528585936594, "replicates": 8, "sample_sd": 0.011164756642074367, "standard_error": 0.003947337565954166 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3448275862068965 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3448275862068966, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.1-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05645788780086033, -0.05462096753160733 ], "confidence_level": 0.95, "mean": -0.055603817415292175, "replicates": 8, "sample_sd": 0.0014293592205474371, "standard_error": 0.0005053547988003054 }, "log_loss": { "bootstrap_interval": [ -0.25081455293581045, -0.24166934932527814 ], "confidence_level": 0.95, "mean": -0.24665170091581218, "replicates": 8, "sample_sd": 0.007238450660610252, "standard_error": 0.002559178773700877 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0002018807735253702, 0.00011799796772665169 ], "confidence_level": 0.95, "mean": -1.1600466213098465e-05, "replicates": 8, "sample_sd": 0.0002546431929553597, "standard_error": 9.002996426086465e-05 }, "log_loss": { "bootstrap_interval": [ -0.0032372510921028844, 0.0011949669709180083 ], "confidence_level": 0.95, "mean": -0.0012046424939258613, "replicates": 8, "sample_sd": 0.00340033711798006, "standard_error": 0.0012022007172220108 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00012627427948455313, 0.0009979896999678711 ], "confidence_level": 0.95, "mean": 0.0005274616479668728, "replicates": 8, "sample_sd": 0.0006842989933868676, "standard_error": 0.00024193622929149125 }, "log_loss": { "bootstrap_interval": [ 0.002798920707971775, 0.009520117511753162 ], "confidence_level": 0.95, "mean": 0.005824154038773935, "replicates": 8, "sample_sd": 0.005327037709253045, "standard_error": 0.00188339224392464 } } }, "discount": { "bootstrap_interval": [ 0.39549671633381334, 0.9008456693980751 ], "confidence_level": 0.95, "mean": 0.6481711928659442, "replicates": 8, "sample_sd": 0.38663371985017075, "standard_error": 0.1366956625707178 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.20676535518460076, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.059075558077624504, 0.06012111968067104 ], "confidence_level": 0.95, "mean": 0.05957206750239875, "replicates": 8, "sample_sd": 0.0008196992614422926, "standard_error": 0.0002898074531497249 }, "log_loss": { "bootstrap_interval": [ 0.26274363821639396, 0.2655450861407855 ], "confidence_level": 0.95, "mean": 0.2640996976579437, "replicates": 8, "sample_sd": 0.0021245785894268713, "standard_error": 0.0007511519638737452 } }, "independent": { "brier": { "bootstrap_interval": [ 0.002868981093684724, 0.005265848032676463 ], "confidence_level": 0.95, "mean": 0.003979850553319668, "replicates": 8, "sample_sd": 0.0019078980415024587, "standard_error": 0.0006745438214794607 }, "log_loss": { "bootstrap_interval": [ 0.013852339440979449, 0.024401340453250427 ], "confidence_level": 0.95, "mean": 0.01865263923605742, "replicates": 8, "sample_sd": 0.00823264698418251, "standard_error": 0.002910680254815216 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10869681169237655, 0.11376108136650306 ], "confidence_level": 0.95, "mean": 0.11112646541261903, "replicates": 8, "sample_sd": 0.003931336936424466, "standard_error": 0.0013899375034374436 }, "log_loss": { "bootstrap_interval": [ 0.34384548970497913, 0.3574384830089951 ], "confidence_level": 0.95, "mean": 0.3507030921738555, "replicates": 8, "sample_sd": 0.010442164998598529, "standard_error": 0.0036918628403889175 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0029357861355656448, 0.005307230029987353 ], "confidence_level": 0.95, "mean": 0.00396825008710657, "replicates": 8, "sample_sd": 0.0018450833995380196, "standard_error": 0.0006523354918340308 }, "log_loss": { "bootstrap_interval": [ 0.013001235831430391, 0.023458280382630196 ], "confidence_level": 0.95, "mean": 0.017447996742131558, "replicates": 8, "sample_sd": 0.008337394067293787, "standard_error": 0.002947713941203963 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.0027127355999537876, 0.0043091719767675285 ], "confidence_level": 0.95, "mean": 0.003440788439139697, "replicates": 8, "sample_sd": 0.0012400704596885607, "standard_error": 0.0004384311155974502 }, "log_loss": { "bootstrap_interval": [ 0.00952167740446649, 0.014157908403160542 ], "confidence_level": 0.95, "mean": 0.011623842703357623, "replicates": 8, "sample_sd": 0.0035811957770404924, "standard_error": 0.0012661439093509797 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3448275862068965 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3448275862068966, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.25-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04417911043498372, -0.040951531406207345 ], "confidence_level": 0.95, "mean": -0.04252246413854871, "replicates": 8, "sample_sd": 0.002498786477947929, "standard_error": 0.0008834544316471148 }, "log_loss": { "bootstrap_interval": [ -0.13127868340962032, -0.1194939328600733 ], "confidence_level": 0.95, "mean": -0.1253128648147434, "replicates": 8, "sample_sd": 0.009076328947180504, "standard_error": 0.0032089668734155458 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03968197628446067, -0.032697479188720055 ], "confidence_level": 0.95, "mean": -0.03603611972592186, "replicates": 8, "sample_sd": 0.005448204853772238, "standard_error": 0.0019262312986979059 }, "log_loss": { "bootstrap_interval": [ -0.7820477721588346, -0.6947817265362355 ], "confidence_level": 0.95, "mean": -0.7373581175838458, "replicates": 8, "sample_sd": 0.06613319912931567, "standard_error": 0.023381616782949696 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00010719980383678933, 0.00033601546887388964 ], "confidence_level": 0.95, "mean": 9.625834044479573e-05, "replicates": 8, "sample_sd": 0.0003480073472845743, "standard_error": 0.00012303917758383216 }, "log_loss": { "bootstrap_interval": [ -0.0001797281072556528, 0.002963573878055099 ], "confidence_level": 0.95, "mean": 0.0010835596510119089, "replicates": 8, "sample_sd": 0.0025925310531592903, "standard_error": 0.0009165981440628179 } } }, "discount": { "bootstrap_interval": [ 0.16669897792974253, 0.18607827962356593 ], "confidence_level": 0.95, "mean": 0.1755988760490603, "replicates": 8, "sample_sd": 0.015177019010824392, "standard_error": 0.005365886530375537 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.2059897995121937, "discount_min": 0.16027775203028205, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17035812465400257, 0.17212809905761192 ], "confidence_level": 0.95, "mean": 0.1711672134683975, "replicates": 8, "sample_sd": 0.0013820951687113351, "standard_error": 0.0004886444330204753 }, "log_loss": { "bootstrap_interval": [ 0.5253464639431105, 0.5292666939607144 ], "confidence_level": 0.95, "mean": 0.5271620188235568, "replicates": 8, "sample_sd": 0.003046603775958449, "standard_error": 0.0010771370947843801 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15985599898316966, 0.1696065839527337 ], "confidence_level": 0.95, "mean": 0.16468086905577065, "replicates": 8, "sample_sd": 0.007587373235019801, "standard_error": 0.0026825415329379066 }, "log_loss": { "bootstrap_interval": [ 1.0935951416961311, 1.1906970450935859 ], "confidence_level": 0.95, "mean": 1.1392072715926593, "replicates": 8, "sample_sd": 0.07584691833092222, "standard_error": 0.026815935141948678 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19672560065943895, 0.20066872593205412 ], "confidence_level": 0.95, "mean": 0.19864580941794724, "replicates": 8, "sample_sd": 0.0030921164901485903, "standard_error": 0.0010932282692014072 }, "log_loss": { "bootstrap_interval": [ 0.5751539868673414, 0.5843721958241073 ], "confidence_level": 0.95, "mean": 0.579797825332825, "replicates": 8, "sample_sd": 0.007299326707341479, "standard_error": 0.0025807017064286165 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1264612294621947, 0.1310165191780974 ], "confidence_level": 0.95, "mean": 0.1286447493298488, "replicates": 8, "sample_sd": 0.0034698471890627345, "standard_error": 0.00122677623853367 }, "log_loss": { "bootstrap_interval": [ 0.39499161314118797, 0.40935000220210266 ], "confidence_level": 0.95, "mean": 0.40184915400881344, "replicates": 8, "sample_sd": 0.01119529325116598, "standard_error": 0.003958133887635727 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.12643791065329726, 0.13093916567658292 ], "confidence_level": 0.95, "mean": 0.128548490989404, "replicates": 8, "sample_sd": 0.0034822284232566723, "standard_error": 0.001231153665862666 }, "log_loss": { "bootstrap_interval": [ 0.3943647552971971, 0.40827222426316806 ], "confidence_level": 0.95, "mean": 0.40076559435780157, "replicates": 8, "sample_sd": 0.010883629679422248, "standard_error": 0.003847944175121321 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.17391304347826086 ], "oracle_method": "oracle", "population_optimal_scalar": 0.17391304347826086, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.25-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04322714942526527, -0.040522969449773585 ], "confidence_level": 0.95, "mean": -0.04188174093319717, "replicates": 8, "sample_sd": 0.0020936917681876025, "standard_error": 0.0007402318234999534 }, "log_loss": { "bootstrap_interval": [ -0.18760124105487164, -0.17606805854306684 ], "confidence_level": 0.95, "mean": -0.1818754015195299, "replicates": 8, "sample_sd": 0.008889722941968138, "standard_error": 0.0031429916875676477 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.005945369952366183, -0.004359428872208082 ], "confidence_level": 0.95, "mean": -0.005139798160014678, "replicates": 8, "sample_sd": 0.0012384342433691192, "standard_error": 0.0004378526257699676 }, "log_loss": { "bootstrap_interval": [ -0.24302333964556166, -0.1825855071133743 ], "confidence_level": 0.95, "mean": -0.21116600314322193, "replicates": 8, "sample_sd": 0.04642134179203349, "standard_error": 0.01641242278646268 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.454453800176411e-06, 0.00036359413578458807 ], "confidence_level": 0.95, "mean": 0.00017204036452486346, "replicates": 8, "sample_sd": 0.00028180019041745743, "standard_error": 9.963141279192223e-05 }, "log_loss": { "bootstrap_interval": [ 2.3831669594636007e-05, 0.002756929063647443 ], "confidence_level": 0.95, "mean": 0.0013248375266598292, "replicates": 8, "sample_sd": 0.002133390373567673, "standard_error": 0.0007542674000339017 } } }, "discount": { "bootstrap_interval": [ 0.15020280697939165, 0.18164123098688767 ], "confidence_level": 0.95, "mean": 0.1654482102917827, "replicates": 8, "sample_sd": 0.024183394267989444, "standard_error": 0.00855012103950161 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.20728239442003327, "discount_min": 0.1332370448528415, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06710569403057601, 0.06891395449203475 ], "confidence_level": 0.95, "mean": 0.06793605269977548, "replicates": 8, "sample_sd": 0.001421167325130339, "standard_error": 0.0005024585264002048 }, "log_loss": { "bootstrap_interval": [ 0.2713349031172337, 0.2756829366576369 ], "confidence_level": 0.95, "mean": 0.2733228965716519, "replicates": 8, "sample_sd": 0.003364395571084577, "standard_error": 0.0011894934614539456 } }, "independent": { "brier": { "bootstrap_interval": [ 0.02891947240710626, 0.03343560518693593 ], "confidence_level": 0.95, "mean": 0.031194109926592987, "replicates": 8, "sample_sd": 0.0035369834485158304, "standard_error": 0.0012505124906950617 }, "log_loss": { "bootstrap_interval": [ 0.2682908164948388, 0.34021058014535077 ], "confidence_level": 0.95, "mean": 0.3026134981953439, "replicates": 8, "sample_sd": 0.05644634814730786, "standard_error": 0.019956797774089048 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11037104554432721, 0.11429438731392753 ], "confidence_level": 0.95, "mean": 0.11236518873898231, "replicates": 8, "sample_sd": 0.003048391672834064, "standard_error": 0.001077769211786785 }, "log_loss": { "bootstrap_interval": [ 0.3477652020517863, 0.35915704695256695 ], "confidence_level": 0.95, "mean": 0.3538358241699904, "replicates": 8, "sample_sd": 0.00876608933492542, "standard_error": 0.003099280606606418 } }, "learned": { "brier": { "bootstrap_interval": [ 0.024230995137392, 0.027934733004044757 ], "confidence_level": 0.95, "mean": 0.02605431176657831, "replicates": 8, "sample_sd": 0.0028795389246689587, "standard_error": 0.0010180707501620198 }, "log_loss": { "bootstrap_interval": [ 0.08466869632108637, 0.0984567356301213 ], "confidence_level": 0.95, "mean": 0.09144749505212199, "replicates": 8, "sample_sd": 0.01074207106183059, "standard_error": 0.003797895645904093 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.024157047911543664, 0.027750791814033237 ], "confidence_level": 0.95, "mean": 0.025882271402053448, "replicates": 8, "sample_sd": 0.0028113844825163486, "standard_error": 0.0009939745160549714 }, "log_loss": { "bootstrap_interval": [ 0.08358294235024044, 0.09739819711787664 ], "confidence_level": 0.95, "mean": 0.09012265752546215, "replicates": 8, "sample_sd": 0.010614691748339806, "standard_error": 0.0037528602577279833 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.17391304347826086 ], "oracle_method": "oracle", "population_optimal_scalar": 0.17391304347826086, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.5-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.012687209316219068, -0.01031627635126723 ], "confidence_level": 0.95, "mean": -0.01148722728897723, "replicates": 8, "sample_sd": 0.0018300254527960013, "standard_error": 0.0006470117037080173 }, "log_loss": { "bootstrap_interval": [ -0.03771338819219031, -0.027651088052133104 ], "confidence_level": 0.95, "mean": -0.033032456923818496, "replicates": 8, "sample_sd": 0.007904052727634447, "standard_error": 0.0027945046412831726 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06585223048485742, -0.059478233829642936 ], "confidence_level": 0.95, "mean": -0.06257795506511295, "replicates": 8, "sample_sd": 0.0046660575922546745, "standard_error": 0.0016497004824451273 }, "log_loss": { "bootstrap_interval": [ -1.8382607067336385, -1.7045320639128307 ], "confidence_level": 0.95, "mean": -1.7676937073241246, "replicates": 8, "sample_sd": 0.1050846983724848, "standard_error": 0.03715305140906348 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 4.297038865595099e-06, 0.0009032790374402945 ], "confidence_level": 0.95, "mean": 0.00039568462936895657, "replicates": 8, "sample_sd": 0.0007378487691270184, "standard_error": 0.000260868934069931 }, "log_loss": { "bootstrap_interval": [ 5.890748775069787e-05, 0.005131941323829438 ], "confidence_level": 0.95, "mean": 0.0020669361881656967, "replicates": 8, "sample_sd": 0.0042832207614214805, "standard_error": 0.001514347222860068 } } }, "discount": { "bootstrap_interval": [ 0.09069083452956524, 0.10537878870382142 ], "confidence_level": 0.95, "mean": 0.09752330756812398, "replicates": 8, "sample_sd": 0.01124997941090771, "standard_error": 0.003977468364830941 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.11980811123362917, "discount_min": 0.0825178769108589, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17991316839167823, 0.18239210857913501 ], "confidence_level": 0.95, "mean": 0.1810812566533208, "replicates": 8, "sample_sd": 0.0019086733457739614, "standard_error": 0.000674817932933392 }, "log_loss": { "bootstrap_interval": [ 0.5419410092957666, 0.5476440123155512 ], "confidence_level": 0.95, "mean": 0.5445020222123519, "replicates": 8, "sample_sd": 0.004419492490973722, "standard_error": 0.0015625265548852728 } }, "independent": { "brier": { "bootstrap_interval": [ 0.22758293506633237, 0.23627868496610355 ], "confidence_level": 0.95, "mean": 0.23217198442945655, "replicates": 8, "sample_sd": 0.006649660748073652, "standard_error": 0.0023510101037764446 }, "log_loss": { "bootstrap_interval": [ 2.211553394391374, 2.35913023907411 ], "confidence_level": 0.95, "mean": 2.279163272612658, "replicates": 8, "sample_sd": 0.11437026741271816, "standard_error": 0.04043599582682591 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19772911304399343, 0.20074829467201522 ], "confidence_level": 0.95, "mean": 0.19926702311469635, "replicates": 8, "sample_sd": 0.0023311563873632864, "standard_error": 0.0008241882447554569 }, "log_loss": { "bootstrap_interval": [ 0.577971966493757, 0.5850850955533938 ], "confidence_level": 0.95, "mean": 0.581755125408795, "replicates": 8, "sample_sd": 0.005465067828045487, "standard_error": 0.0019321932604277003 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16745815999090377, 0.1718266734378373 ], "confidence_level": 0.95, "mean": 0.1695940293643436, "replicates": 8, "sample_sd": 0.00334397327510755, "standard_error": 0.0011822730894675685 }, "log_loss": { "bootstrap_interval": [ 0.5047602781658518, 0.5186814407606788 ], "confidence_level": 0.95, "mean": 0.5114695652885334, "replicates": 8, "sample_sd": 0.01092720624498877, "standard_error": 0.0038633508176277747 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1672206404118969, 0.17134447143923603 ], "confidence_level": 0.95, "mean": 0.16919834473497464, "replicates": 8, "sample_sd": 0.0031474696449937892, "standard_error": 0.0011127985647769619 }, "log_loss": { "bootstrap_interval": [ 0.5040792421258036, 0.5158865263535277 ], "confidence_level": 0.95, "mean": 0.5094026291003677, "replicates": 8, "sample_sd": 0.00928033280401073, "standard_error": 0.003281093128691977 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.09523809523809523 ], "oracle_method": "oracle", "population_optimal_scalar": 0.09523809523809523, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.5-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.018134307858427935, -0.015541044582954595 ], "confidence_level": 0.95, "mean": -0.016772317815454452, "replicates": 8, "sample_sd": 0.001982908057046418, "standard_error": 0.0007010638668034818 }, "log_loss": { "bootstrap_interval": [ -0.08004734458923003, -0.06576353581619719 ], "confidence_level": 0.95, "mean": -0.07322901634842738, "replicates": 8, "sample_sd": 0.01102981074815245, "standard_error": 0.003899626987611432 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.021565728279753765, -0.015636845333927472 ], "confidence_level": 0.95, "mean": -0.018425757548082372, "replicates": 8, "sample_sd": 0.004583256747090327, "standard_error": 0.0016204259628932835 }, "log_loss": { "bootstrap_interval": [ -1.1263412058004765, -0.9455067434153474 ], "confidence_level": 0.95, "mean": -1.034241458713762, "replicates": 8, "sample_sd": 0.1405229632895918, "standard_error": 0.04968237012724931 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00019420578853924922, 0.0003212524410509931 ], "confidence_level": 0.95, "mean": 3.586656088940248e-05, "replicates": 8, "sample_sd": 0.000381469337821411, "standard_error": 0.0001348697777941308 }, "log_loss": { "bootstrap_interval": [ -0.0009088223537783145, 0.004968485493320695 ], "confidence_level": 0.95, "mean": 0.001372048394388984, "replicates": 8, "sample_sd": 0.004831887715078652, "standard_error": 0.0017083302846320435 } } }, "discount": { "bootstrap_interval": [ 0.08646679397351684, 0.10258740445296967 ], "confidence_level": 0.95, "mean": 0.09356757003815047, "replicates": 8, "sample_sd": 0.012559871360302857, "standard_error": 0.004440585104850428 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.12043627363827347, "discount_min": 0.08052339574963055, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08127065294825606, 0.08414106221342403 ], "confidence_level": 0.95, "mean": 0.0826053068320128, "replicates": 8, "sample_sd": 0.002248187447245238, "standard_error": 0.0007948542946627906 }, "log_loss": { "bootstrap_interval": [ 0.2900733197393007, 0.2971075762966985 ], "confidence_level": 0.95, "mean": 0.2933272175344622, "replicates": 8, "sample_sd": 0.0054399197023119635, "standard_error": 0.001923302055307547 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07941213028997325, 0.08986382424026096 ], "confidence_level": 0.95, "mean": 0.08425874656464072, "replicates": 8, "sample_sd": 0.008058982655960554, "standard_error": 0.0028492806427472405 }, "log_loss": { "bootstrap_interval": [ 1.1585487978795135, 1.3550579820088322 ], "confidence_level": 0.95, "mean": 1.254339659899797, "replicates": 8, "sample_sd": 0.15485596521026285, "standard_error": 0.05474985155368247 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1119571906415926, 0.11542044268566098 ], "confidence_level": 0.95, "mean": 0.11374706502855107, "replicates": 8, "sample_sd": 0.0026832453909882235, "standard_error": 0.0009486705057776608 }, "log_loss": { "bootstrap_interval": [ 0.3527490523771007, 0.3623158885509511 ], "confidence_level": 0.95, "mean": 0.3585218932160255, "replicates": 8, "sample_sd": 0.007593125351894876, "standard_error": 0.0026845752133621784 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06339050336745718, 0.06857305883433448 ], "confidence_level": 0.95, "mean": 0.06583298901655833, "replicates": 8, "sample_sd": 0.0039395974004661875, "standard_error": 0.0013928580185072678 }, "log_loss": { "bootstrap_interval": [ 0.2109535129518254, 0.23026234761080536 ], "confidence_level": 0.95, "mean": 0.22009820118603485, "replicates": 8, "sample_sd": 0.015008979071261907, "standard_error": 0.005306475439988132 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06335385284752121, 0.06855204148756766 ], "confidence_level": 0.95, "mean": 0.06579712245566893, "replicates": 8, "sample_sd": 0.004058358376395922, "standard_error": 0.0014348463642173917 }, "log_loss": { "bootstrap_interval": [ 0.21058829282462882, 0.22794411182543403 ], "confidence_level": 0.95, "mean": 0.21872615279164584, "replicates": 8, "sample_sd": 0.013587413897437188, "standard_error": 0.004803876252833086 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.09523809523809523 ], "oracle_method": "oracle", "population_optimal_scalar": 0.09523809523809523, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.75-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.002175045629387627, -0.00042098548122903136 ], "confidence_level": 0.95, "mean": -0.0013574466763775772, "replicates": 8, "sample_sd": 0.001370560044098307, "standard_error": 0.0004845661506026232 }, "log_loss": { "bootstrap_interval": [ -0.006473293558556595, 0.0012897551274648486 ], "confidence_level": 0.95, "mean": -0.0031390319133996203, "replicates": 8, "sample_sd": 0.0061975463603931216, "standard_error": 0.0021911635290759912 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.08379638988233158, -0.07768482264926826 ], "confidence_level": 0.95, "mean": -0.08095932933327392, "replicates": 8, "sample_sd": 0.0046588956306308675, "standard_error": 0.0016471683466297315 }, "log_loss": { "bootstrap_interval": [ -2.8079149551367193, -2.6337126443940777 ], "confidence_level": 0.95, "mean": -2.7155676319720867, "replicates": 8, "sample_sd": 0.13671147164864766, "standard_error": 0.0483348043343756 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.615000301503092e-05, 0.0012207623301754636 ], "confidence_level": 0.95, "mean": 0.0005343231183196906, "replicates": 8, "sample_sd": 0.0009356228029810628, "standard_error": 0.0003307926143103373 }, "log_loss": { "bootstrap_interval": [ 0.00022457507712507485, 0.005697521686922321 ], "confidence_level": 0.95, "mean": 0.0023231372210474333, "replicates": 8, "sample_sd": 0.004610836850740676, "standard_error": 0.0016301770020517785 } } }, "discount": { "bootstrap_interval": [ 0.061805068861296705, 0.07350385476072802 ], "confidence_level": 0.95, "mean": 0.06730408101827856, "replicates": 8, "sample_sd": 0.008958061135502944, "standard_error": 0.0031671528875988975 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.08482049767526578, "discount_min": 0.05408624902748441, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18951897492628844, 0.19255714565708243 ], "confidence_level": 0.95, "mean": 0.1909921101306646, "replicates": 8, "sample_sd": 0.0023427985967580515, "standard_error": 0.000828304387360973 }, "log_loss": { "bootstrap_interval": [ 0.5608237091299462, 0.5683953107950852 ], "confidence_level": 0.95, "mean": 0.5643147220265121, "replicates": 8, "sample_sd": 0.005752125367763616, "standard_error": 0.002033683426890408 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2657900702977089, 0.27413751865212305 ], "confidence_level": 0.95, "mean": 0.27059399278756097, "replicates": 8, "sample_sd": 0.006592513200342883, "standard_error": 0.0023308053945121404 }, "log_loss": { "bootstrap_interval": [ 3.1893579705465873, 3.376549715872183 ], "confidence_level": 0.95, "mean": 3.276743322085199, "replicates": 8, "sample_sd": 0.14567893489305994, "standard_error": 0.05150528136945812 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1985900288960512, 0.20111158875271906 ], "confidence_level": 0.95, "mean": 0.19980328185793006, "replicates": 8, "sample_sd": 0.001967302994709199, "standard_error": 0.0006955466441037386 }, "log_loss": { "bootstrap_interval": [ 0.5806496509027949, 0.5867897272273161 ], "confidence_level": 0.95, "mean": 0.5837083029857186, "replicates": 8, "sample_sd": 0.004731593149177854, "standard_error": 0.0016728708007997361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18759128761793592, 0.19172236648769067 ], "confidence_level": 0.95, "mean": 0.18963466345428703, "replicates": 8, "sample_sd": 0.0032210372162794223, "standard_error": 0.0011388086290427097 }, "log_loss": { "bootstrap_interval": [ 0.5550860156963788, 0.5680648882259666 ], "confidence_level": 0.95, "mean": 0.5611756901131125, "replicates": 8, "sample_sd": 0.010131341883812008, "standard_error": 0.003581970274281381 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18726417219663008, 0.19106659203497697 ], "confidence_level": 0.95, "mean": 0.18910034033596734, "replicates": 8, "sample_sd": 0.0029249166758543678, "standard_error": 0.001034114207951119 }, "log_loss": { "bootstrap_interval": [ 0.554080365251619, 0.5643773505378363 ], "confidence_level": 0.95, "mean": 0.558852552892065, "replicates": 8, "sample_sd": 0.00808360581749187, "standard_error": 0.002857986244993763 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.06557377049180328 ], "oracle_method": "oracle", "population_optimal_scalar": 0.06557377049180328, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho0.75-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.003909452860086443, -0.002586945870614438 ], "confidence_level": 0.95, "mean": -0.0032123866193469325, "replicates": 8, "sample_sd": 0.0010171815364130746, "standard_error": 0.00035962798104771806 }, "log_loss": { "bootstrap_interval": [ -0.018120889115737417, -0.0027291378888613937 ], "confidence_level": 0.95, "mean": -0.0118852931851492, "replicates": 8, "sample_sd": 0.012552735966088888, "standard_error": 0.00443806236203286 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.035187820875594925, -0.027275869517073403 ], "confidence_level": 0.95, "mean": -0.031149044214881176, "replicates": 8, "sample_sd": 0.00605100440664108, "standard_error": 0.0021393531244627943 }, "log_loss": { "bootstrap_interval": [ -2.251833720176648, -1.975980676414105 ], "confidence_level": 0.95, "mean": -2.1075900272951444, "replicates": 8, "sample_sd": 0.21374947908083483, "standard_error": 0.07557185306657518 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.436915939391204e-05, 0.0007483837753351364 ], "confidence_level": 0.95, "mean": 0.00028588004208187495, "replicates": 8, "sample_sd": 0.0006553453066352192, "standard_error": 0.00023169955517027038 }, "log_loss": { "bootstrap_interval": [ -0.0005451583215447652, 0.010738375007483654 ], "confidence_level": 0.95, "mean": 0.0036790212396223126, "replicates": 8, "sample_sd": 0.009677588635143345, "standard_error": 0.003421544274721862 } } }, "discount": { "bootstrap_interval": [ 0.05984262897329052, 0.07501948502242371 ], "confidence_level": 0.95, "mean": 0.06655497998631817, "replicates": 8, "sample_sd": 0.011945925894568128, "standard_error": 0.004223522603800548 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.09193261687277846, "discount_min": 0.05782753782860092, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09670931682470765, 0.100582254598832 ], "confidence_level": 0.95, "mean": 0.09849930599340218, "replicates": 8, "sample_sd": 0.00300605834590836, "standard_error": 0.0010628021205171087 }, "log_loss": { "bootstrap_interval": [ 0.31793233296417084, 0.3284629158624253 ], "confidence_level": 0.95, "mean": 0.32278994160940594, "replicates": 8, "sample_sd": 0.008313955258632861, "standard_error": 0.002939427070930426 } }, "independent": { "brier": { "bootstrap_interval": [ 0.12218916194039617, 0.13234579194410143 ], "confidence_level": 0.95, "mean": 0.12643596358893644, "replicates": 8, "sample_sd": 0.008105721507004475, "standard_error": 0.0028658053220062523 }, "log_loss": { "bootstrap_interval": [ 2.2773239614817555, 2.5719513424033495 ], "confidence_level": 0.95, "mean": 2.4184946757194012, "replicates": 8, "sample_sd": 0.22883647301460652, "standard_error": 0.08090591092572033 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11269027749958911, 0.11676135510180255 ], "confidence_level": 0.95, "mean": 0.11469046688997823, "replicates": 8, "sample_sd": 0.003156091016768506, "standard_error": 0.0011158466799994781 }, "log_loss": { "bootstrap_interval": [ 0.3570419212786541, 0.36870703754060785 ], "confidence_level": 0.95, "mean": 0.3628740273563677, "replicates": 8, "sample_sd": 0.008938420049611378, "standard_error": 0.0031602087150870008 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09294884710469306, 0.0978827025777209 ], "confidence_level": 0.95, "mean": 0.09528691937405526, "replicates": 8, "sample_sd": 0.0038741418345912193, "standard_error": 0.0013697159812589715 }, "log_loss": { "bootstrap_interval": [ 0.30028195155115556, 0.3236430241649993 ], "confidence_level": 0.95, "mean": 0.31090464842425675, "replicates": 8, "sample_sd": 0.018397424269040584, "standard_error": 0.006504471728502279 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.0927238030305054, 0.09778005403646313 ], "confidence_level": 0.95, "mean": 0.09500103933197337, "replicates": 8, "sample_sd": 0.0039065940263047485, "standard_error": 0.0013811895636714727 }, "log_loss": { "bootstrap_interval": [ 0.29933909322486474, 0.3164064761163412 ], "confidence_level": 0.95, "mean": 0.3072256271846344, "replicates": 8, "sample_sd": 0.013285465891446477, "standard_error": 0.004697121511532192 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.06557377049180328 ], "oracle_method": "oracle", "population_optimal_scalar": 0.06557377049180328, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho1-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 8.425078805292599e-05, 0.0015625145189691637 ], "confidence_level": 0.95, "mean": 0.0007045402269900553, "replicates": 8, "sample_sd": 0.0012227781525520922, "standard_error": 0.0004323173617781715 }, "log_loss": { "bootstrap_interval": [ 0.00026492837470714416, 0.0067574748425747915 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0963660722279362, -0.09180286270763618 ], "confidence_level": 0.95, "mean": -0.09394972437086521, "replicates": 8, "sample_sd": 0.003562557676688318, "standard_error": 0.0012595543457772506 }, "log_loss": { "bootstrap_interval": [ -3.6772774854737844, -3.466611519809474 ], "confidence_level": 0.95, "mean": -3.5630802522660283, "replicates": 8, "sample_sd": 0.16163662341940294, "standard_error": 0.05714717625397806 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.425078805292599e-05, 0.0015625145189691533 ], "confidence_level": 0.95, "mean": 0.0007045402269900518, "replicates": 8, "sample_sd": 0.001222778152552083, "standard_error": 0.0004323173617781683 }, "log_loss": { "bootstrap_interval": [ 0.00026492837470714416, 0.0067574748425747915 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } } }, "discount": { "bootstrap_interval": [ 0.04637531665961638, 0.05711109337171089 ], "confidence_level": 0.95, "mean": 0.051487970674181927, "replicates": 8, "sample_sd": 0.008329514899395478, "standard_error": 0.0029449282346784627 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.0676943818372854, "discount_min": 0.03906576467106912, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19897052260664516, 0.20255274223276593 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.0027431450244216854, "standard_error": 0.0009698482242733556 }, "log_loss": { "bootstrap_interval": [ 0.5822836877206246, 0.5914080214359457 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2922738083773679, 0.29810939603279685 ], "confidence_level": 0.95, "mean": 0.2953854067882206, "replicates": 8, "sample_sd": 0.004539995939274454, "standard_error": 0.0016051309576101778 }, "log_loss": { "bootstrap_interval": [ 4.04989727019563, 4.27082124087449 ], "confidence_level": 0.95, "mean": 4.152445890979787, "replicates": 8, "sample_sd": 0.1704616698206003, "standard_error": 0.06026730133126436 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19897309424389742, 0.20254050216783043 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5822296918902574, 0.5914087729844594 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1994248459273742, 0.20346045733519402 ], "confidence_level": 0.95, "mean": 0.20143568241735543, "replicates": 8, "sample_sd": 0.00314778408765836, "standard_error": 0.001112909737047168 }, "log_loss": { "bootstrap_interval": [ 0.583382013854901, 0.5960028293614309 ], "confidence_level": 0.95, "mean": 0.5893656387137585, "replicates": 8, "sample_sd": 0.009854990310042909, "standard_error": 0.0034842652383795286 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19897197028795965, 0.2025861333345745 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5822982570509598, 0.5915615241376294 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.05 ], "oracle_method": "oracle", "population_optimal_scalar": 0.05, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n20-rho1-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 7.887237571566047e-05, 0.001628872271821434 ], "confidence_level": 0.95, "mean": 0.0007233667431064527, "replicates": 8, "sample_sd": 0.0011947528903027155, "standard_error": 0.0004224089352876387 }, "log_loss": { "bootstrap_interval": [ 0.00010550320098887701, 0.016165689362724774 ], "confidence_level": 0.95, "mean": 0.006085127988558439, "replicates": 8, "sample_sd": 0.013870011259328706, "standard_error": 0.004903789508302546 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.046487517320471315, -0.03792904116283068 ], "confidence_level": 0.95, "mean": -0.041955902271181116, "replicates": 8, "sample_sd": 0.006597226906772641, "standard_error": 0.0023324719414026425 }, "log_loss": { "bootstrap_interval": [ -3.4101593337612117, -3.0709108746849467 ], "confidence_level": 0.95, "mean": -3.227623132737108, "replicates": 8, "sample_sd": 0.26314500059198337, "standard_error": 0.09303580717696475 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 7.887237571565705e-05, 0.0016288722718214323 ], "confidence_level": 0.95, "mean": 0.000723366743106451, "replicates": 8, "sample_sd": 0.0011947528903027164, "standard_error": 0.00042240893528763904 }, "log_loss": { "bootstrap_interval": [ 0.00010550320098885656, 0.016165689362724774 ], "confidence_level": 0.95, "mean": 0.006085127988558432, "replicates": 8, "sample_sd": 0.013870011259328708, "standard_error": 0.004903789508302547 } } }, "discount": { "bootstrap_interval": [ 0.045867774164287084, 0.05979527170406965 ], "confidence_level": 0.95, "mean": 0.05201354278247175, "replicates": 8, "sample_sd": 0.010885528250855779, "standard_error": 0.003848615421488929 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.07454593604939189, "discount_min": 0.042852866135966214, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.11366131334243566, 0.11831207312709628 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.3600859100265078, 0.3762278874665715 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15315178199115326, 0.1645659537492242 ], "confidence_level": 0.95, "mean": 0.15850191664067848, "replicates": 8, "sample_sd": 0.008879924030228189, "standard_error": 0.0031395272490978643 }, "log_loss": { "bootstrap_interval": [ 3.4351338514050576, 3.7932916393247 ], "confidence_level": 0.95, "mean": 3.601450994457287, "replicates": 8, "sample_sd": 0.28082193612098144, "standard_error": 0.09928554766854072 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11365334380340447, 0.11836791499210668 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.36022249605145085, 0.37634107594305993 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11402088025920741, 0.11933324567553165 ], "confidence_level": 0.95, "mean": 0.11654601436949735, "replicates": 8, "sample_sd": 0.004073392697855985, "standard_error": 0.0014401617995448662 }, "log_loss": { "bootstrap_interval": [ 0.3620855816962216, 0.3900729460798864 ], "confidence_level": 0.95, "mean": 0.373827861720179, "replicates": 8, "sample_sd": 0.022116164976176075, "standard_error": 0.00781924511424726 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1136754964477846, 0.11834174259200436 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.003638834271105444, "standard_error": 0.0012865221943563336 }, "log_loss": { "bootstrap_interval": [ 0.36019555401325754, 0.3763480057663502 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.05 ], "oracle_method": "oracle", "population_optimal_scalar": 0.05, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07598289409022103, -0.07139824540249674 ], "confidence_level": 0.95, "mean": -0.07362294619135869, "replicates": 8, "sample_sd": 0.0035914363638773644, "standard_error": 0.0012697645035488205 }, "log_loss": { "bootstrap_interval": [ -0.2262918121093887, -0.2111024681327802 ], "confidence_level": 0.95, "mean": -0.21879657323793453, "replicates": 8, "sample_sd": 0.011947178159935472, "standard_error": 0.004223965346467095 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 9.495121259684575e-07, 0.0003427273640548203 ], "confidence_level": 0.95, "mean": 0.00011564360913130477, "replicates": 8, "sample_sd": 0.0003203131209384785, "standard_error": 0.00011324778995931241 }, "log_loss": { "bootstrap_interval": [ 5.7486657289024246e-06, 0.002488646738858416 ], "confidence_level": 0.95, "mean": 0.000904793528130679, "replicates": 8, "sample_sd": 0.0022140473121662115, "standard_error": 0.0007827839341502885 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 9.495121259684575e-07, 0.0003427273640548203 ], "confidence_level": 0.95, "mean": 0.00011564360913130477, "replicates": 8, "sample_sd": 0.0003203131209384785, "standard_error": 0.00011324778995931241 }, "log_loss": { "bootstrap_interval": [ 5.7486657289024246e-06, 0.002488646738858416 ], "confidence_level": 0.95, "mean": 0.000904793528130679, "replicates": 8, "sample_sd": 0.0022140473121662115, "standard_error": 0.0007827839341502885 } } }, "discount": { "bootstrap_interval": [ 0.9181083279907503, 0.9990933169635994 ], "confidence_level": 0.95, "mean": 0.9680689011573049, "replicates": 8, "sample_sd": 0.06981748937184937, "standard_error": 0.0246842100901272 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8001577073337811, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16468611296663682, 0.16730219534869023 ], "confidence_level": 0.95, "mean": 0.16602648942540713, "replicates": 8, "sample_sd": 0.0020156438739972727, "standard_error": 0.0007126377258802972 }, "log_loss": { "bootstrap_interval": [ 0.5154934933832205, 0.5210059766303293 ], "confidence_level": 0.95, "mean": 0.5182950730431581, "replicates": 8, "sample_sd": 0.004207258928717438, "standard_error": 0.001487490659351875 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08887762535173456, 0.09560398419799787 ], "confidence_level": 0.95, "mean": 0.09228789962491715, "replicates": 8, "sample_sd": 0.005146853715253066, "standard_error": 0.0018196875819153093 }, "log_loss": { "bootstrap_interval": [ 0.2891705802685748, 0.3082183333242667 ], "confidence_level": 0.95, "mean": 0.298593706277093, "replicates": 8, "sample_sd": 0.01495257620300486, "standard_error": 0.005286534014676667 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19467963926357856, 0.20090172517798832 ], "confidence_level": 0.95, "mean": 0.19761153359360628, "replicates": 8, "sample_sd": 0.004840249158008265, "standard_error": 0.0017112865011300604 }, "log_loss": { "bootstrap_interval": [ 0.5705039078972285, 0.5846279882266521 ], "confidence_level": 0.95, "mean": 0.5772891634083406, "replicates": 8, "sample_sd": 0.01097636370028827, "standard_error": 0.0038807306026218498 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08896091048358266, 0.09571790221228363 ], "confidence_level": 0.95, "mean": 0.09240354323404845, "replicates": 8, "sample_sd": 0.005256340859982556, "standard_error": 0.001858397133160797 }, "log_loss": { "bootstrap_interval": [ 0.28981608333784653, 0.30973766672904096 ], "confidence_level": 0.95, "mean": 0.29949849980522364, "replicates": 8, "sample_sd": 0.015526773034180433, "standard_error": 0.005489543251206705 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08893490803250073, 0.09551733153186404 ], "confidence_level": 0.95, "mean": 0.09228789962491715, "replicates": 8, "sample_sd": 0.005146853715253066, "standard_error": 0.0018196875819153093 }, "log_loss": { "bootstrap_interval": [ 0.28900884828100765, 0.30848624961618787 ], "confidence_level": 0.95, "mean": 0.298593706277093, "replicates": 8, "sample_sd": 0.01495257620300486, "standard_error": 0.005286534014676667 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05436320195348353, -0.05217530958990526 ], "confidence_level": 0.95, "mean": -0.05326265481909814, "replicates": 8, "sample_sd": 0.001693317457372981, "standard_error": 0.0005986781284049987 }, "log_loss": { "bootstrap_interval": [ -0.23444457982642503, -0.227211249317925 ], "confidence_level": 0.95, "mean": -0.2309913251253041, "replicates": 8, "sample_sd": 0.005569654240145807, "standard_error": 0.001969170141035754 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -2.8253052929692532e-05, 9.431472002604862e-05 ], "confidence_level": 0.95, "mean": 2.7591415820084515e-05, "replicates": 8, "sample_sd": 9.254159887143273e-05, "standard_error": 3.2718396051917716e-05 }, "log_loss": { "bootstrap_interval": [ 9.37240475008148e-05, 0.00065407442329703 ], "confidence_level": 0.95, "mean": 0.00035802167683116016, "replicates": 8, "sample_sd": 0.0004315134044272644, "standard_error": 0.0001525630272217059 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -2.8253052929692532e-05, 9.431472002604862e-05 ], "confidence_level": 0.95, "mean": 2.7591415820084515e-05, "replicates": 8, "sample_sd": 9.254159887143273e-05, "standard_error": 3.2718396051917716e-05 }, "log_loss": { "bootstrap_interval": [ 9.37240475008148e-05, 0.00065407442329703 ], "confidence_level": 0.95, "mean": 0.00035802167683116016, "replicates": 8, "sample_sd": 0.0004315134044272644, "standard_error": 0.0001525630272217059 } } }, "discount": { "bootstrap_interval": [ 0.8188291879862422, 0.9670353985007573 ], "confidence_level": 0.95, "mean": 0.8971618131153808, "replicates": 8, "sample_sd": 0.11355176159308882, "standard_error": 0.040146610319075626 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.686866337127767, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0615753383699615, 0.06332570338538412 ], "confidence_level": 0.95, "mean": 0.062436706433771014, "replicates": 8, "sample_sd": 0.0013549996862191604, "standard_error": 0.0004790647333156062 }, "log_loss": { "bootstrap_interval": [ 0.26281824745915466, 0.2672163930156263 ], "confidence_level": 0.95, "mean": 0.26501408847996766, "replicates": 8, "sample_sd": 0.003444178026692772, "standard_error": 0.0012177008191440804 } }, "independent": { "brier": { "bootstrap_interval": [ 0.008350620395045734, 0.010298502406476876 ], "confidence_level": 0.95, "mean": 0.009146460198852792, "replicates": 8, "sample_sd": 0.0015913665279271587, "standard_error": 0.0005626330316252925 }, "log_loss": { "bootstrap_interval": [ 0.030200484883693333, 0.0381130132213999 ], "confidence_level": 0.95, "mean": 0.03366474167783243, "replicates": 8, "sample_sd": 0.00624009525421605, "standard_error": 0.002206206834753081 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10676816022736377, 0.11300392670287578 ], "confidence_level": 0.95, "mean": 0.10971962570460989, "replicates": 8, "sample_sd": 0.004901062078073617, "standard_error": 0.0017327871152110436 }, "log_loss": { "bootstrap_interval": [ 0.3401941950829188, 0.35585814527172244 ], "confidence_level": 0.95, "mean": 0.3478498156750285, "replicates": 8, "sample_sd": 0.01248682896093513, "standard_error": 0.0044147607168969006 } }, "learned": { "brier": { "bootstrap_interval": [ 0.008377239692539132, 0.010336178265614289 ], "confidence_level": 0.95, "mean": 0.009174051614672878, "replicates": 8, "sample_sd": 0.001592306061974084, "standard_error": 0.0005629652070731608 }, "log_loss": { "bootstrap_interval": [ 0.03052595389620986, 0.03844056727507473 ], "confidence_level": 0.95, "mean": 0.03402276335466359, "replicates": 8, "sample_sd": 0.006209889155063529, "standard_error": 0.0021955273659811105 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.008344863678820978, 0.010313804381077572 ], "confidence_level": 0.95, "mean": 0.009146460198852792, "replicates": 8, "sample_sd": 0.0015913665279271587, "standard_error": 0.0005626330316252925 }, "log_loss": { "bootstrap_interval": [ 0.03014257378967331, 0.038157804866178355 ], "confidence_level": 0.95, "mean": 0.03366474167783243, "replicates": 8, "sample_sd": 0.00624009525421605, "standard_error": 0.002206206834753081 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.1-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04694673157339464, -0.044689650521430084 ], "confidence_level": 0.95, "mean": -0.04587243425924424, "replicates": 8, "sample_sd": 0.0017647962418073, "standard_error": 0.0006239496949972379 }, "log_loss": { "bootstrap_interval": [ -0.13937380056257567, -0.130537642886381 ], "confidence_level": 0.95, "mean": -0.13511704389943296, "replicates": 8, "sample_sd": 0.006674647054631352, "standard_error": 0.0023598440971783225 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.003500674564852759, -0.002400027168962586 ], "confidence_level": 0.95, "mean": -0.0029853542108241784, "replicates": 8, "sample_sd": 0.0008703694111639325, "standard_error": 0.0003077220563856795 }, "log_loss": { "bootstrap_interval": [ -0.021849694631399692, -0.015699890890664883 ], "confidence_level": 0.95, "mean": -0.01913951059012551, "replicates": 8, "sample_sd": 0.004802437777332921, "standard_error": 0.0016979181592892795 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.817181241831956e-06, 0.0010285581303362962 ], "confidence_level": 0.95, "mean": 0.00040918282038850985, "replicates": 8, "sample_sd": 0.0008551295474046215, "standard_error": 0.00030233395088139555 }, "log_loss": { "bootstrap_interval": [ 0.00013732040196975476, 0.005334415480772216 ], "confidence_level": 0.95, "mean": 0.002109713913039206, "replicates": 8, "sample_sd": 0.0044370176055039765, "standard_error": 0.0015687226185479796 } } }, "discount": { "bootstrap_interval": [ 0.662420803951827, 0.7844201082464403 ], "confidence_level": 0.95, "mean": 0.7171880279538758, "replicates": 8, "sample_sd": 0.09619822084470572, "standard_error": 0.03401120714868625 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.9193166012187644, "discount_min": 0.6048184000259227, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16883330939924196, 0.17087609775870682 ], "confidence_level": 0.95, "mean": 0.16985378479405444, "replicates": 8, "sample_sd": 0.001609137636408507, "standard_error": 0.0005689160672834742 }, "log_loss": { "bootstrap_interval": [ 0.5222312741081355, 0.5266847603934035 ], "confidence_level": 0.95, "mean": 0.5244975303424706, "replicates": 8, "sample_sd": 0.0034523791694122266, "standard_error": 0.001220600360959283 } }, "independent": { "brier": { "bootstrap_interval": [ 0.12492348677219922, 0.12895378707547997 ], "confidence_level": 0.95, "mean": 0.12696670474563437, "replicates": 8, "sample_sd": 0.0031044482964443314, "standard_error": 0.001097588221129406 }, "log_loss": { "bootstrap_interval": [ 0.40283690160477237, 0.4141485930249515 ], "confidence_level": 0.95, "mean": 0.4085199970331631, "replicates": 8, "sample_sd": 0.008953390765650469, "standard_error": 0.0031655016625022304 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1958081888891329, 0.20070208727326852 ], "confidence_level": 0.95, "mean": 0.1981676486512557, "replicates": 8, "sample_sd": 0.0038049404201851056, "standard_error": 0.0013452495865618396 }, "log_loss": { "bootstrap_interval": [ 0.5730144300090945, 0.5843118108875289 ], "confidence_level": 0.95, "mean": 0.578515600100781, "replicates": 8, "sample_sd": 0.008863064931613176, "standard_error": 0.00313356665762018 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1219322025550797, 0.1261199639941296 ], "confidence_level": 0.95, "mean": 0.1239813505348102, "replicates": 8, "sample_sd": 0.003247460513836996, "standard_error": 0.0011481506754848448 }, "log_loss": { "bootstrap_interval": [ 0.38325142615542857, 0.395887036252047 ], "confidence_level": 0.95, "mean": 0.3893804864430376, "replicates": 8, "sample_sd": 0.009732397985837322, "standard_error": 0.003440922306495933 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.12178927195317175, 0.12536979958376168 ], "confidence_level": 0.95, "mean": 0.1235721677144217, "replicates": 8, "sample_sd": 0.0027935825330697886, "standard_error": 0.0009876805764689701 }, "log_loss": { "bootstrap_interval": [ 0.3823665258758677, 0.3920301032423926 ], "confidence_level": 0.95, "mean": 0.38727077252999836, "replicates": 8, "sample_sd": 0.007370527117275946, "standard_error": 0.0026058748527725785 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.7142857142857143 ], "oracle_method": "oracle", "population_optimal_scalar": 0.7142857142857143, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.1-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04435296752286134, -0.04309153176138081 ], "confidence_level": 0.95, "mean": -0.04363745461770702, "replicates": 8, "sample_sd": 0.0009974419732168675, "standard_error": 0.00035264899155086887 }, "log_loss": { "bootstrap_interval": [ -0.1914138789559448, -0.18574820730087124 ], "confidence_level": 0.95, "mean": -0.18840572717918616, "replicates": 8, "sample_sd": 0.004361473872179054, "standard_error": 0.0015420138754928792 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0009409612678188874, -0.00023920721438970604 ], "confidence_level": 0.95, "mean": -0.0005934437904162324, "replicates": 8, "sample_sd": 0.0005349972239469474, "standard_error": 0.00018915008248443225 }, "log_loss": { "bootstrap_interval": [ -0.008315491132519219, -0.004466282614535421 ], "confidence_level": 0.95, "mean": -0.006335693774928134, "replicates": 8, "sample_sd": 0.002951533425788206, "standard_error": 0.0010435246501368008 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -6.660754847642725e-05, 5.157069975857984e-05 ], "confidence_level": 0.95, "mean": -5.264233787587863e-06, "replicates": 8, "sample_sd": 9.02703092266858e-05, "standard_error": 3.191537389699805e-05 }, "log_loss": { "bootstrap_interval": [ 5.782922833955927e-06, 0.0010872855194394519 ], "confidence_level": 0.95, "mean": 0.0005169237299035845, "replicates": 8, "sample_sd": 0.0008323794964541773, "standard_error": 0.00029429059323169625 } } }, "discount": { "bootstrap_interval": [ 0.6549393658749155, 0.7449581796807485 ], "confidence_level": 0.95, "mean": 0.6982380218222006, "replicates": 8, "sample_sd": 0.06943949498937783, "standard_error": 0.024550568894579176 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8282119952841553, "discount_min": 0.5983360970435356, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06721215236486501, 0.06835736744416505 ], "confidence_level": 0.95, "mean": 0.06778069630600413, "replicates": 8, "sample_sd": 0.0009000649427057228, "standard_error": 0.00031822101224774897 }, "log_loss": { "bootstrap_interval": [ 0.2699853734895047, 0.2732980605290872 ], "confidence_level": 0.95, "mean": 0.2716414560258366, "replicates": 8, "sample_sd": 0.0025336531488281064, "standard_error": 0.0008957816613555014 } }, "independent": { "brier": { "bootstrap_interval": [ 0.023504020117124567, 0.02582058086162498 ], "confidence_level": 0.95, "mean": 0.024736685478713338, "replicates": 8, "sample_sd": 0.001799986072479328, "standard_error": 0.0006363911789457366 }, "log_loss": { "bootstrap_interval": [ 0.08501926608654817, 0.09459969551557536 ], "confidence_level": 0.95, "mean": 0.08957142262157858, "replicates": 8, "sample_sd": 0.00743223092473735, "standard_error": 0.0026276904431130725 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10866877274009713, 0.11370453468602036 ], "confidence_level": 0.95, "mean": 0.11112646541261903, "replicates": 8, "sample_sd": 0.003931336936424466, "standard_error": 0.0013899375034374436 }, "log_loss": { "bootstrap_interval": [ 0.3437900897703434, 0.35752294907254156 ], "confidence_level": 0.95, "mean": 0.3507030921738555, "replicates": 8, "sample_sd": 0.010442164998598529, "standard_error": 0.0036918628403889175 } }, "learned": { "brier": { "bootstrap_interval": [ 0.023193499451967088, 0.024944983324988578 ], "confidence_level": 0.95, "mean": 0.024143241688297106, "replicates": 8, "sample_sd": 0.00137181836353329, "standard_error": 0.0004850110337053109 }, "log_loss": { "bootstrap_interval": [ 0.08044850860772934, 0.08628384445093303 ], "confidence_level": 0.95, "mean": 0.08323572884665045, "replicates": 8, "sample_sd": 0.004563635268525919, "standard_error": 0.0016134887226183841 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.023205898657208658, 0.024998891048917796 ], "confidence_level": 0.95, "mean": 0.024148505922084694, "replicates": 8, "sample_sd": 0.0013823311721259004, "standard_error": 0.0004887278728278864 }, "log_loss": { "bootstrap_interval": [ 0.0797882212014958, 0.0861635109683698 ], "confidence_level": 0.95, "mean": 0.08271880511674687, "replicates": 8, "sample_sd": 0.004854767772975349, "standard_error": 0.0017164196066783912 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.7142857142857143 ], "oracle_method": "oracle", "population_optimal_scalar": 0.7142857142857143, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.25-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.02522396802637232, -0.02383748780766601 ], "confidence_level": 0.95, "mean": -0.02461813889518987, "replicates": 8, "sample_sd": 0.001084503453191465, "standard_error": 0.00038342987298595617 }, "log_loss": { "bootstrap_interval": [ -0.0742772573950956, -0.06876522144318714 ], "confidence_level": 0.95, "mean": -0.07155337825412075, "replicates": 8, "sample_sd": 0.004202309838688135, "standard_error": 0.0014857408917916635 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.012191886956791721, -0.010800112633082882 ], "confidence_level": 0.95, "mean": -0.011586364749568041, "replicates": 8, "sample_sd": 0.0010874403789382279, "standard_error": 0.0003844682330416449 }, "log_loss": { "bootstrap_interval": [ -0.0953034391846569, -0.08786132965917409 ], "confidence_level": 0.95, "mean": -0.09150967849626043, "replicates": 8, "sample_sd": 0.005840117599120586, "standard_error": 0.0020647933786325325 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.3921665525388344e-05, 0.0005723994227298561 ], "confidence_level": 0.95, "mean": 0.0002526135885145553, "replicates": 8, "sample_sd": 0.0004514681777452541, "standard_error": 0.00015961810498680136 }, "log_loss": { "bootstrap_interval": [ -5.125435188906317e-06, 0.0026098297786686583 ], "confidence_level": 0.95, "mean": 0.0011262162454401467, "replicates": 8, "sample_sd": 0.0020682865426590834, "standard_error": 0.0007312497198755586 } } }, "discount": { "bootstrap_interval": [ 0.4676529853734401, 0.5304776488117433 ], "confidence_level": 0.95, "mean": 0.49968736018722076, "replicates": 8, "sample_sd": 0.04831145931443356, "standard_error": 0.01708068024512698 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.581078301762672, "discount_min": 0.40926188713461586, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.174064734231072, 0.17607926168143878 ], "confidence_level": 0.95, "mean": 0.17510726470133228, "replicates": 8, "sample_sd": 0.0015577322279440294, "standard_error": 0.0005507415108260259 }, "log_loss": { "bootstrap_interval": [ 0.5311335782680873, 0.53549581667182 ], "confidence_level": 0.95, "mean": 0.5333887696103211, "replicates": 8, "sample_sd": 0.003399623332538118, "standard_error": 0.0012019483559588561 } }, "independent": { "brier": { "bootstrap_interval": [ 0.16053086902403652, 0.16383758436963727 ], "confidence_level": 0.95, "mean": 0.16207549055571044, "replicates": 8, "sample_sd": 0.002645303186835399, "standard_error": 0.0009352559108528476 }, "log_loss": { "bootstrap_interval": [ 0.5465732845045473, 0.5598662714412073 ], "confidence_level": 0.95, "mean": 0.5533450698524608, "replicates": 8, "sample_sd": 0.01041503108854487, "standard_error": 0.003682269554489393 } }, "individual": { "brier": { "bootstrap_interval": [ 0.196692049221322, 0.2006517402313589 ], "confidence_level": 0.95, "mean": 0.19864580941794724, "replicates": 8, "sample_sd": 0.0030921164901485903, "standard_error": 0.0010932282692014072 }, "log_loss": { "bootstrap_interval": [ 0.5750827932966014, 0.5844314961751869 ], "confidence_level": 0.95, "mean": 0.579797825332825, "replicates": 8, "sample_sd": 0.007299326707341479, "standard_error": 0.0025807017064286165 } }, "learned": { "brier": { "bootstrap_interval": [ 0.14903470515421186, 0.15208683482315588 ], "confidence_level": 0.95, "mean": 0.1504891258061424, "replicates": 8, "sample_sd": 0.002390254826579175, "standard_error": 0.0008450826983190049 }, "log_loss": { "bootstrap_interval": [ 0.4572692765116066, 0.4663808721821259 ], "confidence_level": 0.95, "mean": 0.46183539135620033, "replicates": 8, "sample_sd": 0.007066421567997789, "standard_error": 0.002498357304727056 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.14888922291630186, 0.15159898975294941 ], "confidence_level": 0.95, "mean": 0.15023651221762785, "replicates": 8, "sample_sd": 0.002085352395780172, "standard_error": 0.0007372834101098863 }, "log_loss": { "bootstrap_interval": [ 0.45656177279341675, 0.46453026875255876 ], "confidence_level": 0.95, "mean": 0.4607091751107602, "replicates": 8, "sample_sd": 0.0060830876875801135, "standard_error": 0.0021506962772201463 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.25-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.02909989276219353, -0.027216723812714866 ], "confidence_level": 0.95, "mean": -0.028104476026661548, "replicates": 8, "sample_sd": 0.0014617736059001143, "standard_error": 0.0005168150146457413 }, "log_loss": { "bootstrap_interval": [ -0.1294245315168488, -0.12012352997569627 ], "confidence_level": 0.95, "mean": -0.12459032045894583, "replicates": 8, "sample_sd": 0.007104278436141944, "standard_error": 0.0025117417288166646 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0050220768080814875, -0.0040454989848020895 ], "confidence_level": 0.95, "mean": -0.004562941914989645, "replicates": 8, "sample_sd": 0.0007534334513167853, "standard_error": 0.0002663789512994417 }, "log_loss": { "bootstrap_interval": [ -0.06115548975314757, -0.04549857370695157 ], "confidence_level": 0.95, "mean": -0.053310341603658766, "replicates": 8, "sample_sd": 0.012012497836601712, "standard_error": 0.004247059339624901 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00015808922899069634, 0.00018276681109813484 ], "confidence_level": 0.95, "mean": -5.145285864810792e-07, "replicates": 8, "sample_sd": 0.00026211866377973575, "standard_error": 9.267294231710391e-05 }, "log_loss": { "bootstrap_interval": [ -0.00032880524585109864, 0.0017021232749753583 ], "confidence_level": 0.95, "mean": 0.0005041368949189363, "replicates": 8, "sample_sd": 0.0016336601386325895, "standard_error": 0.0005775860810906296 } } }, "discount": { "bootstrap_interval": [ 0.4658672450521429, 0.5323999331236451 ], "confidence_level": 0.95, "mean": 0.49662566129171365, "replicates": 8, "sample_sd": 0.051817755094571066, "standard_error": 0.018320343006617484 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.6046127796257963, "discount_min": 0.43636706544571335, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07452952521737571, 0.07592818513687831 ], "confidence_level": 0.95, "mean": 0.07521896985648617, "replicates": 8, "sample_sd": 0.0010856792884036603, "standard_error": 0.0003838455935120068 }, "log_loss": { "bootstrap_interval": [ 0.27945910459692586, 0.28329363607207636 ], "confidence_level": 0.95, "mean": 0.28136111612492803, "replicates": 8, "sample_sd": 0.002949204325151674, "standard_error": 0.001042701188709722 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04986250222204515, 0.05344364680907981 ], "confidence_level": 0.95, "mean": 0.05167743574481427, "replicates": 8, "sample_sd": 0.002765529494643785, "standard_error": 0.0009777623296170132 }, "log_loss": { "bootstrap_interval": [ 0.19720046426543641, 0.22350089070369403 ], "confidence_level": 0.95, "mean": 0.21008113726964098, "replicates": 8, "sample_sd": 0.020284483861495384, "standard_error": 0.007171648045666235 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11036725909907538, 0.11426898906618088 ], "confidence_level": 0.95, "mean": 0.11236518873898231, "replicates": 8, "sample_sd": 0.003048391672834064, "standard_error": 0.001077769211786785 }, "log_loss": { "bootstrap_interval": [ 0.3478488144995887, 0.35908880673980514 ], "confidence_level": 0.95, "mean": 0.3538358241699904, "replicates": 8, "sample_sd": 0.00876608933492542, "standard_error": 0.003099280606606418 } }, "learned": { "brier": { "bootstrap_interval": [ 0.04570126201887453, 0.04852889572302567 ], "confidence_level": 0.95, "mean": 0.04711449382982462, "replicates": 8, "sample_sd": 0.0022150594752803014, "standard_error": 0.0007831417878511084 }, "log_loss": { "bootstrap_interval": [ 0.15140959286616978, 0.162376630110721 ], "confidence_level": 0.95, "mean": 0.1567707956659822, "replicates": 8, "sample_sd": 0.00842604808707408, "standard_error": 0.0029790578704870095 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.04559885814911814, 0.048619841447810286 ], "confidence_level": 0.95, "mean": 0.047115008358411106, "replicates": 8, "sample_sd": 0.0023348483878937254, "standard_error": 0.0008254935640610658 }, "log_loss": { "bootstrap_interval": [ 0.15062244016789422, 0.16230703069713104 ], "confidence_level": 0.95, "mean": 0.15626665877106327, "replicates": 8, "sample_sd": 0.008998576354586767, "standard_error": 0.003181477180676613 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.5-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.008033291610837611, -0.007061666706237778 ], "confidence_level": 0.95, "mean": -0.007560492448343045, "replicates": 8, "sample_sd": 0.0007685211038910494, "standard_error": 0.0002717132420231661 }, "log_loss": { "bootstrap_interval": [ -0.0239622290463659, -0.01808150560824355 ], "confidence_level": 0.95, "mean": -0.021211671422790748, "replicates": 8, "sample_sd": 0.004569913854937466, "standard_error": 0.0016157085381323193 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.028018717554984526, -0.024593545295237818 ], "confidence_level": 0.95, "mean": -0.026444710882694757, "replicates": 8, "sample_sd": 0.002722426460362127, "standard_error": 0.0009625231057018747 }, "log_loss": { "bootstrap_interval": [ -0.2506385891393372, -0.2323338755394031 ], "confidence_level": 0.95, "mean": -0.24162175705175326, "replicates": 8, "sample_sd": 0.01400016804023694, "standard_error": 0.004949806879501359 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 5.4604099392327465e-05, 0.0005453857791062096 ], "confidence_level": 0.95, "mean": 0.0002770619454910486, "replicates": 8, "sample_sd": 0.00038466084347488206, "standard_error": 0.0001359981454390131 }, "log_loss": { "bootstrap_interval": [ 9.441728141171728e-05, 0.0030925816563105307 ], "confidence_level": 0.95, "mean": 0.0014243605763221284, "replicates": 8, "sample_sd": 0.0023522395556192896, "standard_error": 0.0008316422703768154 } } }, "discount": { "bootstrap_interval": [ 0.3107361527009782, 0.357801365386204 ], "confidence_level": 0.95, "mean": 0.3358096476257831, "replicates": 8, "sample_sd": 0.03657943484959937, "standard_error": 0.012932783217061615 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.38898801140812667, "discount_min": 0.2623707334451074, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18257844223888667, 0.18483346116677413 ], "confidence_level": 0.95, "mean": 0.1836918960230254, "replicates": 8, "sample_sd": 0.0017591232612417592, "standard_error": 0.0006219439934835212 }, "log_loss": { "bootstrap_interval": [ 0.5465859830241531, 0.5518904244373766 ], "confidence_level": 0.95, "mean": 0.549267189847859, "replicates": 8, "sample_sd": 0.00408040013861261, "standard_error": 0.0014426393039837523 } }, "independent": { "brier": { "bootstrap_interval": [ 0.20027174581210822, 0.2048985227015953 ], "confidence_level": 0.95, "mean": 0.20257611445737714, "replicates": 8, "sample_sd": 0.003579844622925518, "standard_error": 0.0012656662042324165 }, "log_loss": { "bootstrap_interval": [ 0.7568772592297554, 0.7824772917238878 ], "confidence_level": 0.95, "mean": 0.7696772754768216, "replicates": 8, "sample_sd": 0.020075079008183852, "standard_error": 0.007097612249771255 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19774016292159527, 0.20075543885147515 ], "confidence_level": 0.95, "mean": 0.19926702311469635, "replicates": 8, "sample_sd": 0.0023311563873632864, "standard_error": 0.0008241882447554569 }, "log_loss": { "bootstrap_interval": [ 0.578002208587859, 0.5851353324750057 ], "confidence_level": 0.95, "mean": 0.581755125408795, "replicates": 8, "sample_sd": 0.005465067828045487, "standard_error": 0.0019321932604277003 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17465022147839185, 0.1776253024342997 ], "confidence_level": 0.95, "mean": 0.17613140357468238, "replicates": 8, "sample_sd": 0.002304539153660431, "standard_error": 0.0008147776315315989 }, "log_loss": { "bootstrap_interval": [ 0.5230323108799697, 0.533076937182804 ], "confidence_level": 0.95, "mean": 0.5280555184250684, "replicates": 8, "sample_sd": 0.007677540280849281, "standard_error": 0.0027144203977106985 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.17445073057184632, 0.17734720466231682 ], "confidence_level": 0.95, "mean": 0.17585434162919134, "replicates": 8, "sample_sd": 0.0022371725901381354, "standard_error": 0.0007909599545856741 }, "log_loss": { "bootstrap_interval": [ 0.5224346229894563, 0.5309471173887073 ], "confidence_level": 0.95, "mean": 0.5266311578487463, "replicates": 8, "sample_sd": 0.006764696896904717, "standard_error": 0.0023916815242364603 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.5-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.012075494314089469, -0.010550586804879941 ], "confidence_level": 0.95, "mean": -0.011172214907646964, "replicates": 8, "sample_sd": 0.0012406501858607773, "standard_error": 0.0004386360797512531 }, "log_loss": { "bootstrap_interval": [ -0.054790428683319174, -0.04326887110109437 ], "confidence_level": 0.95, "mean": -0.04929451936864787, "replicates": 8, "sample_sd": 0.008776886338460166, "standard_error": 0.0031030979238143752 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.012911795009355714, -0.01026738034276005 ], "confidence_level": 0.95, "mean": -0.011475067364463855, "replicates": 8, "sample_sd": 0.0020646899725864157, "standard_error": 0.0007299781403318607 }, "log_loss": { "bootstrap_interval": [ -0.21135260047286178, -0.1758265202493371 ], "confidence_level": 0.95, "mean": -0.19376442575660446, "replicates": 8, "sample_sd": 0.027424115136660824, "standard_error": 0.009695888890586755 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -7.528786597677667e-05, 0.00037216756033340084 ], "confidence_level": 0.95, "mean": 0.00013519699365924752, "replicates": 8, "sample_sd": 0.0003514579946663408, "standard_error": 0.0001242591656653975 }, "log_loss": { "bootstrap_interval": [ -0.0005707810795818229, 0.0037785293353049254 ], "confidence_level": 0.95, "mean": 0.0013157286506784716, "replicates": 8, "sample_sd": 0.0034614682445726144, "standard_error": 0.0012238138342995951 } } }, "discount": { "bootstrap_interval": [ 0.3105689758573523, 0.360245138855277 ], "confidence_level": 0.95, "mean": 0.33504640538671926, "replicates": 8, "sample_sd": 0.038982146888338565, "standard_error": 0.013782270204977135 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3883159928523623, "discount_min": 0.28245460875406136, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08676379519011336, 0.0891001160419709 ], "confidence_level": 0.95, "mean": 0.0879038476930216, "replicates": 8, "sample_sd": 0.0018264194490927622, "standard_error": 0.0006457367888722452 }, "log_loss": { "bootstrap_interval": [ 0.29819020889950965, 0.3046971873546789 ], "confidence_level": 0.95, "mean": 0.30137373248776855, "replicates": 8, "sample_sd": 0.005016904251578194, "standard_error": 0.001773743508427281 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08589411079732782, 0.0904834616420246 ], "confidence_level": 0.95, "mean": 0.0882067001498385, "replicates": 8, "sample_sd": 0.0035322195945253296, "standard_error": 0.0012488282139644289 }, "log_loss": { "bootstrap_interval": [ 0.4196674577482818, 0.4706039135187281 ], "confidence_level": 0.95, "mean": 0.44584363887572515, "replicates": 8, "sample_sd": 0.039234079925436746, "standard_error": 0.013871341984445658 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11190096776645816, 0.11539305761456664 ], "confidence_level": 0.95, "mean": 0.11374706502855107, "replicates": 8, "sample_sd": 0.0026832453909882235, "standard_error": 0.0009486705057776608 }, "log_loss": { "bootstrap_interval": [ 0.35282584070864453, 0.3623273160335936 ], "confidence_level": 0.95, "mean": 0.3585218932160255, "replicates": 8, "sample_sd": 0.007593125351894876, "standard_error": 0.0026845752133621784 } }, "learned": { "brier": { "bootstrap_interval": [ 0.07502222446941025, 0.07833158101718309 ], "confidence_level": 0.95, "mean": 0.07673163278537465, "replicates": 8, "sample_sd": 0.002592710281475592, "standard_error": 0.0009166615108417368 }, "log_loss": { "bootstrap_interval": [ 0.24428471287954145, 0.2605948474731202 ], "confidence_level": 0.95, "mean": 0.2520792131191207, "replicates": 8, "sample_sd": 0.012378267856823905, "standard_error": 0.0043763785704518275 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.0748338699231786, 0.07826529099188481 ], "confidence_level": 0.95, "mean": 0.07659643579171539, "replicates": 8, "sample_sd": 0.0026555890814770763, "standard_error": 0.0009388925237786978 }, "log_loss": { "bootstrap_interval": [ 0.24351550047296971, 0.2580589586825979 ], "confidence_level": 0.95, "mean": 0.2507634844684422, "replicates": 8, "sample_sd": 0.011223118497862011, "standard_error": 0.003967971597949203 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.75-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0014043226800737017, -0.00024783833733477145 ], "confidence_level": 0.95, "mean": -0.0008712501560872027, "replicates": 8, "sample_sd": 0.0008843558998848935, "standard_error": 0.0003126670268954698 }, "log_loss": { "bootstrap_interval": [ -0.004246407540582025, 0.002078232335115087 ], "confidence_level": 0.95, "mean": -0.0015792082253080209, "replicates": 8, "sample_sd": 0.0050947103951146915, "standard_error": 0.0018012521342835965 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04244983073239978, -0.037948666028851684 ], "confidence_level": 0.95, "mean": -0.04026366108953702, "replicates": 8, "sample_sd": 0.003495525125320439, "standard_error": 0.0012358547599610195 }, "log_loss": { "bootstrap_interval": [ -0.41397399752073627, -0.3841875973175925 ], "confidence_level": 0.95, "mean": -0.3989144766789543, "replicates": 8, "sample_sd": 0.023394200791706305, "standard_error": 0.008271099010127613 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 3.8025991116154984e-05, 0.0009923879886783243 ], "confidence_level": 0.95, "mean": 0.00044791094663704234, "replicates": 8, "sample_sd": 0.0007510263776140746, "standard_error": 0.0002655279222304404 }, "log_loss": { "bootstrap_interval": [ 6.291357627145888e-05, 0.004913205054324354 ], "confidence_level": 0.95, "mean": 0.002029655205974326, "replicates": 8, "sample_sd": 0.004064914902407623, "standard_error": 0.0014371644462193415 } } }, "discount": { "bootstrap_interval": [ 0.2319965566489701, 0.2767230374226868 ], "confidence_level": 0.95, "mean": 0.25434450619279925, "replicates": 8, "sample_sd": 0.03417906483899465, "standard_error": 0.012084124261133903 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.31750758122780187, "discount_min": 0.1926213401104745, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19092642711251828, 0.19362460019273545 ], "confidence_level": 0.95, "mean": 0.19223066861281085, "replicates": 8, "sample_sd": 0.002100345327664298, "standard_error": 0.0007425842120124531 }, "log_loss": { "bootstrap_interval": [ 0.5636000829528411, 0.5704687063831685 ], "confidence_level": 0.95, "mean": 0.5669188559308831, "replicates": 8, "sample_sd": 0.005287304554713699, "standard_error": 0.0018693444524182877 } }, "independent": { "brier": { "bootstrap_interval": [ 0.22857048462458374, 0.23449899704992372 ], "confidence_level": 0.95, "mean": 0.23162307954626066, "replicates": 8, "sample_sd": 0.004570673293755675, "standard_error": 0.0016159770403014451 }, "log_loss": { "bootstrap_interval": [ 0.9456638911139725, 0.9845516399974974 ], "confidence_level": 0.95, "mean": 0.9642541243845294, "replicates": 8, "sample_sd": 0.030371153740321098, "standard_error": 0.010737824381120111 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19855944361568373, 0.20111158953786604 ], "confidence_level": 0.95, "mean": 0.19980328185793006, "replicates": 8, "sample_sd": 0.001967302994709199, "standard_error": 0.0006955466441037386 }, "log_loss": { "bootstrap_interval": [ 0.5806902933140153, 0.5867894293061829 ], "confidence_level": 0.95, "mean": 0.5837083029857186, "replicates": 8, "sample_sd": 0.004731593149177854, "standard_error": 0.0016728708007997361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18970189978689148, 0.19300857935066532 ], "confidence_level": 0.95, "mean": 0.19135941845672363, "replicates": 8, "sample_sd": 0.0025466300701246164, "standard_error": 0.0009003696958793446 }, "log_loss": { "bootstrap_interval": [ 0.5600648755736203, 0.5712754354151551 ], "confidence_level": 0.95, "mean": 0.5653396477055751, "replicates": 8, "sample_sd": 0.008669860467803681, "standard_error": 0.003065258564362578 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18938478283043844, 0.19249204378748794 ], "confidence_level": 0.95, "mean": 0.1909115075100866, "replicates": 8, "sample_sd": 0.0024266097732212034, "standard_error": 0.0008579361129691316 }, "log_loss": { "bootstrap_interval": [ 0.5589972988581366, 0.5679173276872458 ], "confidence_level": 0.95, "mean": 0.5633099924996008, "replicates": 8, "sample_sd": 0.0069042453361410384, "standard_error": 0.002441019348080461 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25 ], "oracle_method": "oracle", "population_optimal_scalar": 0.25, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho0.75-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0027962474504700623, -0.0019088192982832915 ], "confidence_level": 0.95, "mean": -0.0022792593870421055, "replicates": 8, "sample_sd": 0.0007261103140096632, "standard_error": 0.0002567187634628631 }, "log_loss": { "bootstrap_interval": [ -0.012733742414823387, -0.0008611810569388897 ], "confidence_level": 0.95, "mean": -0.00781916228918473, "replicates": 8, "sample_sd": 0.009469465707935125, "standard_error": 0.003347961708147199 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.021313791223306004, -0.017874299113998078 ], "confidence_level": 0.95, "mean": -0.019645120755105043, "replicates": 8, "sample_sd": 0.0026668259619072898, "standard_error": 0.000942865360954491 }, "log_loss": { "bootstrap_interval": [ -0.39908820147835466, -0.34725137408284573 ], "confidence_level": 0.95, "mean": -0.3736912604655758, "replicates": 8, "sample_sd": 0.04046494517182775, "standard_error": 0.014306518565670622 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -5.634984791433441e-05, 0.0006111292961075397 ], "confidence_level": 0.95, "mean": 0.0002450325614847034, "replicates": 8, "sample_sd": 0.0005101826104789776, "standard_error": 0.00018037679175657 }, "log_loss": { "bootstrap_interval": [ -0.0005262540304549201, 0.007852486309445919 ], "confidence_level": 0.95, "mean": 0.002754428261595339, "replicates": 8, "sample_sd": 0.007002675240270798, "standard_error": 0.0024758195744213085 } } }, "discount": { "bootstrap_interval": [ 0.23001500365509997, 0.2800040372427802 ], "confidence_level": 0.95, "mean": 0.25354207904155707, "replicates": 8, "sample_sd": 0.03907421523402016, "standard_error": 0.013814821280759176 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3255500089112069, "discount_min": 0.21621299793907034, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09977923848982954, 0.10311815473597374 ], "confidence_level": 0.95, "mean": 0.10133385284481244, "replicates": 8, "sample_sd": 0.0025905605918348407, "standard_error": 0.0009159014807805258 }, "log_loss": { "bootstrap_interval": [ 0.3238488628374072, 0.3341994843692626 ], "confidence_level": 0.95, "mean": 0.32891096021953714, "replicates": 8, "sample_sd": 0.008136438766967811, "standard_error": 0.002876665513416025 } }, "independent": { "brier": { "bootstrap_interval": [ 0.11537102990509272, 0.12169295233560488 ], "confidence_level": 0.95, "mean": 0.11869971421287538, "replicates": 8, "sample_sd": 0.004839072327062927, "standard_error": 0.0017108704285591812 }, "log_loss": { "bootstrap_interval": [ 0.6603295170299646, 0.7292045286382822 ], "confidence_level": 0.95, "mean": 0.6947830583959282, "replicates": 8, "sample_sd": 0.05388788644867097, "standard_error": 0.01905224496583295 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11267011664654886, 0.11675907989734108 ], "confidence_level": 0.95, "mean": 0.11469046688997823, "replicates": 8, "sample_sd": 0.003156091016768506, "standard_error": 0.0011158466799994781 }, "log_loss": { "bootstrap_interval": [ 0.3567905104036109, 0.36863570016903274 ], "confidence_level": 0.95, "mean": 0.3628740273563677, "replicates": 8, "sample_sd": 0.008938420049611378, "standard_error": 0.0031602087150870008 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09724443742106856, 0.10094476673061847 ], "confidence_level": 0.95, "mean": 0.09905459345777033, "replicates": 8, "sample_sd": 0.0029215686772120037, "standard_error": 0.0010329305116794096 }, "log_loss": { "bootstrap_interval": [ 0.31214806806346235, 0.33173597627628415 ], "confidence_level": 0.95, "mean": 0.3210917979303524, "replicates": 8, "sample_sd": 0.015490820579403432, "standard_error": 0.0054768321389201445 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.09695376732852926, 0.10079285327068321 ], "confidence_level": 0.95, "mean": 0.09880956089628563, "replicates": 8, "sample_sd": 0.002984181778416825, "standard_error": 0.001055067585905934 }, "log_loss": { "bootstrap_interval": [ 0.3107921371579603, 0.326398838781814 ], "confidence_level": 0.95, "mean": 0.3183373696687571, "replicates": 8, "sample_sd": 0.011895248714570318, "standard_error": 0.004205605514986617 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25 ], "oracle_method": "oracle", "population_optimal_scalar": 0.25, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho1-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 8.681276778124344e-05, 0.0015926425504299152 ], "confidence_level": 0.95, "mean": 0.0007045402269900553, "replicates": 8, "sample_sd": 0.0012227781525520922, "standard_error": 0.0004323173617781715 }, "log_loss": { "bootstrap_interval": [ 0.00029483226502791116, 0.006732992629675444 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.055689953512288996, -0.05128442873063736 ], "confidence_level": 0.95, "mean": -0.053481940556737856, "replicates": 8, "sample_sd": 0.0034081552427952157, "standard_error": 0.0012049648417584906 }, "log_loss": { "bootstrap_interval": [ -0.5745146690470574, -0.5320089667931629 ], "confidence_level": 0.95, "mean": -0.551515924069898, "replicates": 8, "sample_sd": 0.03282341385876121, "standard_error": 0.011604829260611276 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 8.681276778124344e-05, 0.0015926425504299048 ], "confidence_level": 0.95, "mean": 0.0007045402269900518, "replicates": 8, "sample_sd": 0.001222778152552083, "standard_error": 0.0004323173617781683 }, "log_loss": { "bootstrap_interval": [ 0.00029483226502791116, 0.006732992629675444 ], "confidence_level": 0.95, "mean": 0.002788853694967966, "replicates": 8, "sample_sd": 0.00548053676583297, "standard_error": 0.0019376623558313413 } } }, "discount": { "bootstrap_interval": [ 0.18590796043164298, 0.2289333409798151 ], "confidence_level": 0.95, "mean": 0.2059518826967277, "replicates": 8, "sample_sd": 0.03331805959758191, "standard_error": 0.011779712938713851 }, "discount_below_exchangeable_range_count": 4, "discount_max": 0.2707775273491416, "discount_min": 0.15626305868427648, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19903602534402182, 0.20253471292468905 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.0027431450244216854, "standard_error": 0.0009698482242733556 }, "log_loss": { "bootstrap_interval": [ 0.5822362811218099, 0.5913350066452391 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2513552228002509, 0.2579892952884569 ], "confidence_level": 0.95, "mean": 0.25491762297409326, "replicates": 8, "sample_sd": 0.005249740202373476, "standard_error": 0.0018560634482829613 }, "log_loss": { "bootstrap_interval": [ 1.116117557980602, 1.1686725713116393 ], "confidence_level": 0.95, "mean": 1.1408815627836566, "replicates": 8, "sample_sd": 0.041624401259407785, "standard_error": 0.014716448196678555 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1989918006774526, 0.2025608519737126 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5821816479224009, 0.5913572925393936 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19945821321038, 0.20347783965141575 ], "confidence_level": 0.95, "mean": 0.20143568241735543, "replicates": 8, "sample_sd": 0.00314778408765836, "standard_error": 0.001112909737047168 }, "log_loss": { "bootstrap_interval": [ 0.5835336129372571, 0.5961410010639501 ], "confidence_level": 0.95, "mean": 0.5893656387137585, "replicates": 8, "sample_sd": 0.009854990310042909, "standard_error": 0.0034842652383795286 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.198983056420057, 0.20258109819100964 ], "confidence_level": 0.95, "mean": 0.20073114219036536, "replicates": 8, "sample_sd": 0.002743145024421687, "standard_error": 0.0009698482242733562 }, "log_loss": { "bootstrap_interval": [ 0.5823552147380253, 0.5915560325662658 ], "confidence_level": 0.95, "mean": 0.5865767850187905, "replicates": 8, "sample_sd": 0.007165476888891047, "standard_error": 0.0025333786492851723 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2 ], "oracle_method": "oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 }, "exchangeable-n5-rho1-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 7.619469915247047e-05, 0.0015807783501992929 ], "confidence_level": 0.95, "mean": 0.0007233667431064527, "replicates": 8, "sample_sd": 0.0011947528903027155, "standard_error": 0.0004224089352876387 }, "log_loss": { "bootstrap_interval": [ 0.00010577440953596922, 0.016181972361674374 ], "confidence_level": 0.95, "mean": 0.006085127988558432, "replicates": 8, "sample_sd": 0.013870011259328708, "standard_error": 0.004903789508302547 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.029598711507547067, -0.02359426277045184 ], "confidence_level": 0.95, "mean": -0.02650398956098854, "replicates": 8, "sample_sd": 0.004715384928556114, "standard_error": 0.0016671403294434362 }, "log_loss": { "bootstrap_interval": [ -0.5994088323713246, -0.5322524665182262 ], "confidence_level": 0.95, "mean": -0.5634391341116501, "replicates": 8, "sample_sd": 0.05242742836711082, "standard_error": 0.01853589505927801 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 7.619469915246708e-05, 0.0015807783501992929 ], "confidence_level": 0.95, "mean": 0.000723366743106451, "replicates": 8, "sample_sd": 0.0011947528903027164, "standard_error": 0.00042240893528763904 }, "log_loss": { "bootstrap_interval": [ 0.00010577440953596922, 0.016181972361674374 ], "confidence_level": 0.95, "mean": 0.006085127988558432, "replicates": 8, "sample_sd": 0.013870011259328708, "standard_error": 0.004903789508302547 } } }, "discount": { "bootstrap_interval": [ 0.1832309135878114, 0.2382367180307293 ], "confidence_level": 0.95, "mean": 0.208054171129887, "replicates": 8, "sample_sd": 0.043542113003423116, "standard_error": 0.015394461685955716 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.29818374419756755, "discount_min": 0.17141146454386486, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.11367619539738484, 0.11833097768231515 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.3601684976680757, 0.37629694405945796 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1389227030905523, 0.14780635595247996 ], "confidence_level": 0.95, "mean": 0.1430500039304859, "replicates": 8, "sample_sd": 0.0069595400326496, "standard_error": 0.002460568975512889 }, "log_loss": { "bootstrap_interval": [ 0.8959611040653308, 0.9843959221809566 ], "confidence_level": 0.95, "mean": 0.9372669958318292, "replicates": 8, "sample_sd": 0.06905474955887475, "standard_error": 0.024414540843109542 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11367547014157414, 0.11831098982673428 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.0036388342711054463, "standard_error": 0.0012865221943563344 }, "log_loss": { "bootstrap_interval": [ 0.36031215711411124, 0.3763567728248697 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1140474638588625, 0.11930626880743755 ], "confidence_level": 0.95, "mean": 0.11654601436949735, "replicates": 8, "sample_sd": 0.004073392697855985, "standard_error": 0.0014401617995448662 }, "log_loss": { "bootstrap_interval": [ 0.3621410969647962, 0.38988489527341186 ], "confidence_level": 0.95, "mean": 0.373827861720179, "replicates": 8, "sample_sd": 0.022116164976176075, "standard_error": 0.00781924511424726 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11363133374510023, 0.11833713444990425 ], "confidence_level": 0.95, "mean": 0.1158226476263909, "replicates": 8, "sample_sd": 0.003638834271105444, "standard_error": 0.0012865221943563336 }, "log_loss": { "bootstrap_interval": [ 0.36029937489757236, 0.37625500874368867 ], "confidence_level": 0.95, "mean": 0.3677427337316206, "replicates": 8, "sample_sd": 0.01253145279419743, "standard_error": 0.004430537624448056 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2 ], "oracle_method": "oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 200 } }, "config": { "conditions": [ { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a1", "overlap": 1, "signal_strength": 1.0 } ], "phase": "exploratory-stress-exchangeable-m200-v1", "replicates": 8, "schema_version": 1, "seed": 84173001, "test_count": 1000, "train_count": 200 }, "config_sha256": "e129bac73e168927afd6f446a765ea3cc572ab260d28d74fec9adf5c4bafa18e", "expected_runs": 384, "input_path": "results/stress-v1/stress-exchangeable-m200-v1.json", "input_sha256": "ae8b91c3def817d3d84bc2a11a49f37ef7dc02391b39c29ec304197e6d6a1bb2", "ledger_receipt": { "bytes": 131244272, "count": 460800, "payload_sha256": "c02c25d81e3b5bea0231fa74069d10603c65eb5a68f23297150246d70e7bcae4", "schema_version": 1, "sha256": "054d7c8537c54fac42fd50fe8d6767fb2bc19ae7a62c362e161bd2e72438ca5d" }, "manifest_sha256": null, "observed_runs": 384, "phase": "exploratory-stress-exchangeable-m200-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" }, "exploratory-stress-exchangeable-m50-v1": { "analysis_seed": 35193834178639617359134597074429192700169557843501229707551897990644169540172, "cells": { "exchangeable-n10-rho0-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12347445459288788, -0.11927363292849866 ], "confidence_level": 0.95, "mean": -0.12104804740584812, "replicates": 8, "sample_sd": 0.003305968662417007, "standard_error": 0.0011688364297926428 }, "log_loss": { "bootstrap_interval": [ -0.38214010395773496, -0.3655781364908482 ], "confidence_level": 0.95, "mean": -0.37286375829719265, "replicates": 8, "sample_sd": 0.012919740410913953, "standard_error": 0.004567818027863563 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 1.5127499124970717e-06, 0.0007859965854947557 ], "confidence_level": 0.95, "mean": 0.0002852105050219038, "replicates": 8, "sample_sd": 0.0007003022238812697, "standard_error": 0.0002475942256932328 }, "log_loss": { "bootstrap_interval": [ 5.757388290415777e-06, 0.006352323079150544 ], "confidence_level": 0.95, "mean": 0.0022260776458240435, "replicates": 8, "sample_sd": 0.005786124761072874, "standard_error": 0.0020457040276730103 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.5127499124970717e-06, 0.0007859965854947557 ], "confidence_level": 0.95, "mean": 0.0002852105050219038, "replicates": 8, "sample_sd": 0.0007003022238812697, "standard_error": 0.0002475942256932328 }, "log_loss": { "bootstrap_interval": [ 5.757388290415777e-06, 0.006352323079150544 ], "confidence_level": 0.95, "mean": 0.0022260776458240435, "replicates": 8, "sample_sd": 0.005786124761072874, "standard_error": 0.0020457040276730103 } } }, "discount": { "bootstrap_interval": [ 0.8600762888475231, 0.9905127574364613 ], "confidence_level": 0.95, "mean": 0.9399297816538004, "replicates": 8, "sample_sd": 0.11096065435460349, "standard_error": 0.03923051556951837 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.680586028774891, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16063640833352125, 0.16256633533929163 ], "confidence_level": 0.95, "mean": 0.16161833125721622, "replicates": 8, "sample_sd": 0.001508306153786754, "standard_error": 0.0005332667547240066 }, "log_loss": { "bootstrap_interval": [ 0.5094456696187359, 0.5134371430950266 ], "confidence_level": 0.95, "mean": 0.5114909553689748, "replicates": 8, "sample_sd": 0.003122727278070073, "standard_error": 0.001104050817059779 } }, "independent": { "brier": { "bootstrap_interval": [ 0.03715221160625496, 0.04269658388887216 ], "confidence_level": 0.95, "mean": 0.040285073346346206, "replicates": 8, "sample_sd": 0.004416667419014416, "standard_error": 0.00156152774111539 }, "log_loss": { "bootstrap_interval": [ 0.12573223861863023, 0.1452790512722678 ], "confidence_level": 0.95, "mean": 0.13640111942595812, "replicates": 8, "sample_sd": 0.015158546830225583, "standard_error": 0.005359355628293177 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19762404718694263, 0.20330055580686884 ], "confidence_level": 0.95, "mean": 0.200401419084981, "replicates": 8, "sample_sd": 0.004402965745277611, "standard_error": 0.00155668346790894 }, "log_loss": { "bootstrap_interval": [ 0.5787019006454168, 0.5917325343679927 ], "confidence_level": 0.95, "mean": 0.5850450001139459, "replicates": 8, "sample_sd": 0.01030599781392939, "standard_error": 0.0036437204705616027 } }, "learned": { "brier": { "bootstrap_interval": [ 0.03727378174121632, 0.04285345347006736 ], "confidence_level": 0.95, "mean": 0.040570283851368114, "replicates": 8, "sample_sd": 0.004412911359943728, "standard_error": 0.0015601997736956795 }, "log_loss": { "bootstrap_interval": [ 0.12760226787948625, 0.14709892467670532 ], "confidence_level": 0.95, "mean": 0.13862719707178214, "replicates": 8, "sample_sd": 0.015251483214161369, "standard_error": 0.005392213601943152 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.03712141415204384, 0.042722277682280124 ], "confidence_level": 0.95, "mean": 0.040285073346346206, "replicates": 8, "sample_sd": 0.004416667419014416, "standard_error": 0.00156152774111539 }, "log_loss": { "bootstrap_interval": [ 0.12565911008132663, 0.1452813491079994 ], "confidence_level": 0.95, "mean": 0.13640111942595812, "replicates": 8, "sample_sd": 0.015158546830225583, "standard_error": 0.005359355628293177 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05626422374660758, -0.05483606022406043 ], "confidence_level": 0.95, "mean": -0.05552968967533116, "replicates": 8, "sample_sd": 0.0011006238537934715, "standard_error": 0.0003891292952765174 }, "log_loss": { "bootstrap_interval": [ -0.25853473776249253, -0.25445942055222137 ], "confidence_level": 0.95, "mean": -0.25641574553437624, "replicates": 8, "sample_sd": 0.0031828340426163725, "standard_error": 0.001125301767462715 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "discount": { "bootstrap_interval": [ 1.0, 1.0 ], "confidence_level": 0.95, "mean": 1.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 1.0, "discount_one_count": 8, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05529715052070788, 0.056676258927918985 ], "confidence_level": 0.95, "mean": 0.05599972404756052, "replicates": 8, "sample_sd": 0.0010781629612177915, "standard_error": 0.00038118817055063446 }, "log_loss": { "bootstrap_interval": [ 0.25645699451441906, 0.2600689568127255 ], "confidence_level": 0.95, "mean": 0.258249866678179, "replicates": 8, "sample_sd": 0.0028198178414811357, "standard_error": 0.000996956158711062 } }, "independent": { "brier": { "bootstrap_interval": [ 9.350060091506544e-05, 0.0009584961949947771 ], "confidence_level": 0.95, "mean": 0.0004700343722293547, "replicates": 8, "sample_sd": 0.0006918447507065343, "standard_error": 0.0002446040573764534 }, "log_loss": { "bootstrap_interval": [ 0.0005363907592860121, 0.0037248309343154325 ], "confidence_level": 0.95, "mean": 0.0018341211438027488, "replicates": 8, "sample_sd": 0.002602373501293065, "standard_error": 0.0009200779749722524 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1121594254828443, 0.11754409262456723 ], "confidence_level": 0.95, "mean": 0.11474074946836885, "replicates": 8, "sample_sd": 0.004152437295533535, "standard_error": 0.0014681082850618453 }, "log_loss": { "bootstrap_interval": [ 0.3551909615638534, 0.3700690116631166 ], "confidence_level": 0.95, "mean": 0.36231777974411405, "replicates": 8, "sample_sd": 0.011648816374946308, "standard_error": 0.004118478525760715 } }, "learned": { "brier": { "bootstrap_interval": [ 9.181588412942477e-05, 0.0009822249101644708 ], "confidence_level": 0.95, "mean": 0.0004700343722293547, "replicates": 8, "sample_sd": 0.0006918447507065343, "standard_error": 0.0002446040573764534 }, "log_loss": { "bootstrap_interval": [ 0.0005218267102973826, 0.0036956682747368037 ], "confidence_level": 0.95, "mean": 0.0018341211438027488, "replicates": 8, "sample_sd": 0.002602373501293065, "standard_error": 0.0009200779749722524 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.00010017818780860827, 0.0009908520286577528 ], "confidence_level": 0.95, "mean": 0.0004700343722293547, "replicates": 8, "sample_sd": 0.0006918447507065343, "standard_error": 0.0002446040573764534 }, "log_loss": { "bootstrap_interval": [ 0.0005137529278265602, 0.0037309478789633336 ], "confidence_level": 0.95, "mean": 0.0018341211438027488, "replicates": 8, "sample_sd": 0.002602373501293065, "standard_error": 0.0009200779749722524 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.1-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07714690646008361, -0.07371661216199987 ], "confidence_level": 0.95, "mean": -0.07550563993046958, "replicates": 8, "sample_sd": 0.002664378346674819, "standard_error": 0.0009419999982901832 }, "log_loss": { "bootstrap_interval": [ -0.22854619386615138, -0.21370730817363606 ], "confidence_level": 0.95, "mean": -0.2216962124335194, "replicates": 8, "sample_sd": 0.01181759867402443, "standard_error": 0.004178152079871913 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.00610466092920308, -0.0018506579254391146 ], "confidence_level": 0.95, "mean": -0.00399870683708415, "replicates": 8, "sample_sd": 0.003330058029727173, "standard_error": 0.0011773533072823987 }, "log_loss": { "bootstrap_interval": [ -0.04628510219537211, -0.024256770856427946 ], "confidence_level": 0.95, "mean": -0.03659113222574362, "replicates": 8, "sample_sd": 0.017143308002706647, "standard_error": 0.006061074670341738 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0005372387669499833, 0.0032923526510348407 ], "confidence_level": 0.95, "mean": 0.0017747563028908484, "replicates": 8, "sample_sd": 0.002168873257846213, "standard_error": 0.0007668124940786083 }, "log_loss": { "bootstrap_interval": [ 0.005276182607996847, 0.02100621424998081 ], "confidence_level": 0.95, "mean": 0.012436421948934781, "replicates": 8, "sample_sd": 0.01259084854309048, "standard_error": 0.00445153719285602 } } }, "discount": { "bootstrap_interval": [ 0.47862521135135033, 0.759292413815944 ], "confidence_level": 0.95, "mean": 0.6121348697352765, "replicates": 8, "sample_sd": 0.21762143447441848, "standard_error": 0.0769407960242026 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.3240492396269049, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1634709200653783, 0.16681863766162858 ], "confidence_level": 0.95, "mean": 0.1651098456456005, "replicates": 8, "sample_sd": 0.0026081179564939466, "standard_error": 0.0009221089465856352 }, "log_loss": { "bootstrap_interval": [ 0.513144015315986, 0.5203834743497299 ], "confidence_level": 0.95, "mean": 0.5166436474966718, "replicates": 8, "sample_sd": 0.005577519828899916, "standard_error": 0.0019719510466087815 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08963950926324883, 0.09733405839648189 ], "confidence_level": 0.95, "mean": 0.09360291255221508, "replicates": 8, "sample_sd": 0.0058870878498632605, "standard_error": 0.0020813998700396213 }, "log_loss": { "bootstrap_interval": [ 0.3173622316768019, 0.34530267899934347 ], "confidence_level": 0.95, "mean": 0.331538567288896, "replicates": 8, "sample_sd": 0.021893610286543696, "standard_error": 0.007740560149135299 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1972574429916381, 0.20360780498405287 ], "confidence_level": 0.95, "mean": 0.20005045972324384, "replicates": 8, "sample_sd": 0.004854666888269692, "standard_error": 0.0017163839385486471 }, "log_loss": { "bootstrap_interval": [ 0.5777473646268401, 0.5919574183778393 ], "confidence_level": 0.95, "mean": 0.5842897509669385, "replicates": 8, "sample_sd": 0.01106838076050973, "standard_error": 0.003913263546255573 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08666098430741642, 0.09249407669738177 ], "confidence_level": 0.95, "mean": 0.08960420571513092, "replicates": 8, "sample_sd": 0.0045069447020474294, "standard_error": 0.0015934455806252605 }, "log_loss": { "bootstrap_interval": [ 0.2862461137299752, 0.30498525011290434 ], "confidence_level": 0.95, "mean": 0.2949474350631524, "replicates": 8, "sample_sd": 0.014362507640722066, "standard_error": 0.005077913273799087 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08477783150346319, 0.09060646360393187 ], "confidence_level": 0.95, "mean": 0.08782944941224008, "replicates": 8, "sample_sd": 0.004569332626843267, "standard_error": 0.0016155030429689072 }, "log_loss": { "bootstrap_interval": [ 0.2731625627137736, 0.2910812403587164 ], "confidence_level": 0.95, "mean": 0.2825110131142176, "replicates": 8, "sample_sd": 0.01382165343129091, "standard_error": 0.004886692434238058 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5263157894736842 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5263157894736842, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.1-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05555448067005385, -0.052712348517604014 ], "confidence_level": 0.95, "mean": -0.05411243111303496, "replicates": 8, "sample_sd": 0.0021879158674609524, "standard_error": 0.0007735450732736434 }, "log_loss": { "bootstrap_interval": [ -0.24287933162218642, -0.22664667807469935 ], "confidence_level": 0.95, "mean": -0.23468349254551368, "replicates": 8, "sample_sd": 0.01268864122166787, "standard_error": 0.0044861121259422545 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0001588041285527763 ], "confidence_level": 0.95, "mean": 6.623370343423313e-05, "replicates": 8, "sample_sd": 0.00012583133073815756, "standard_error": 4.448809362533923e-05 }, "log_loss": { "bootstrap_interval": [ -0.003288483535522388, 0.0 ], "confidence_level": 0.95, "mean": -0.0012473231844333274, "replicates": 8, "sample_sd": 0.0028661881441446034, "standard_error": 0.0010133505364405674 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.279536018883353e-05, 0.0003175560173515105 ], "confidence_level": 0.95, "mean": 9.521510445204115e-05, "replicates": 8, "sample_sd": 0.00032416144306139107, "standard_error": 0.00011460837729396327 }, "log_loss": { "bootstrap_interval": [ 0.00027396126925951673, 0.007392961581202496 ], "confidence_level": 0.95, "mean": 0.003626614303160157, "replicates": 8, "sample_sd": 0.005562442325331517, "standard_error": 0.0019666203441004917 } } }, "discount": { "bootstrap_interval": [ 0.6840285467369186, 1.0 ], "confidence_level": 0.95, "mean": 0.866533118390206, "replicates": 8, "sample_sd": 0.2584579744342727, "standard_error": 0.09137869318710677 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.32456646746959494, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.059279141735491356, 0.06150096731259949 ], "confidence_level": 0.95, "mean": 0.06044067435468522, "replicates": 8, "sample_sd": 0.0017325441650270132, "standard_error": 0.000612546863897893 }, "log_loss": { "bootstrap_interval": [ 0.2591771338849547, 0.2655106067045263 ], "confidence_level": 0.95, "mean": 0.26242696897319046, "replicates": 8, "sample_sd": 0.0049546895219571875, "standard_error": 0.0017517472798249303 } }, "independent": { "brier": { "bootstrap_interval": [ 0.004900290035034172, 0.00783331481396919 ], "confidence_level": 0.95, "mean": 0.00626200953821603, "replicates": 8, "sample_sd": 0.002254743491790138, "standard_error": 0.0007971722064405206 }, "log_loss": { "bootstrap_interval": [ 0.0205871030735424, 0.038113798415943784 ], "confidence_level": 0.95, "mean": 0.028990799612110093, "replicates": 8, "sample_sd": 0.013722712741568362, "standard_error": 0.004851711617919014 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11170924231134069, 0.11722595419149502 ], "confidence_level": 0.95, "mean": 0.11440040546560537, "replicates": 8, "sample_sd": 0.004214884081710997, "standard_error": 0.0014901865580465399 }, "log_loss": { "bootstrap_interval": [ 0.35349057294871916, 0.36852110384532094 ], "confidence_level": 0.95, "mean": 0.3609438993647717, "replicates": 8, "sample_sd": 0.011549219867090432, "standard_error": 0.00408326584271702 } }, "learned": { "brier": { "bootstrap_interval": [ 0.004940087839291263, 0.007847918578514735 ], "confidence_level": 0.95, "mean": 0.006328243241650263, "replicates": 8, "sample_sd": 0.0022991933495168193, "standard_error": 0.0008128876043511774 }, "log_loss": { "bootstrap_interval": [ 0.020103347745574093, 0.03630810817735631 ], "confidence_level": 0.95, "mean": 0.027743476427676765, "replicates": 8, "sample_sd": 0.012584194244047569, "standard_error": 0.004449184542867377 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.005016238058365133, 0.007633115262169072 ], "confidence_level": 0.95, "mean": 0.006233028137198222, "replicates": 8, "sample_sd": 0.002026589691561131, "standard_error": 0.0007165076567928147 }, "log_loss": { "bootstrap_interval": [ 0.01948348952439028, 0.02920768910196168 ], "confidence_level": 0.95, "mean": 0.024116862124516608, "replicates": 8, "sample_sd": 0.007464286178135184, "standard_error": 0.002639023686638203 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5263157894736842 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5263157894736842, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.25-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.040489591069080795, -0.03433032426914416 ], "confidence_level": 0.95, "mean": -0.03736237928052744, "replicates": 8, "sample_sd": 0.00487503251161803, "standard_error": 0.0017235842737349976 }, "log_loss": { "bootstrap_interval": [ -0.12118598056079719, -0.049614655142155836 ], "confidence_level": 0.95, "mean": -0.09349140926360032, "replicates": 8, "sample_sd": 0.05994214668579339, "standard_error": 0.02119274920020162 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.023350131578525175, -0.013376165827016997 ], "confidence_level": 0.95, "mean": -0.019073052618150656, "replicates": 8, "sample_sd": 0.00798913050116842, "standard_error": 0.002824584176580235 }, "log_loss": { "bootstrap_interval": [ -0.2514848604575978, -0.1568058758194568 ], "confidence_level": 0.95, "mean": -0.21427271663728653, "replicates": 8, "sample_sd": 0.0794880585767504, "standard_error": 0.028103272621486857 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00024516093247656245, 0.006433694211689701 ], "confidence_level": 0.95, "mean": 0.0028006136500822454, "replicates": 8, "sample_sd": 0.005140927358303973, "standard_error": 0.0018175922983220917 }, "log_loss": { "bootstrap_interval": [ 0.0010931723445498365, 0.07285068984907853 ], "confidence_level": 0.95, "mean": 0.027004088835285836, "replicates": 8, "sample_sd": 0.0636537199745494, "standard_error": 0.022504988520876734 } } }, "discount": { "bootstrap_interval": [ 0.2727268440980283, 0.5567108745767937 ], "confidence_level": 0.95, "mean": 0.3826547313847357, "replicates": 8, "sample_sd": 0.23397796984352787, "standard_error": 0.08272370456231003 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.9449738082474759, "discount_min": 0.19033232067095218, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16802277119003212, 0.1728061880568715 ], "confidence_level": 0.95, "mean": 0.17041509986658313, "replicates": 8, "sample_sd": 0.0036888976387164725, "standard_error": 0.0013042222677197301 }, "log_loss": { "bootstrap_interval": [ 0.5200140928507285, 0.5301352212625143 ], "confidence_level": 0.95, "mean": 0.5249749004994205, "replicates": 8, "sample_sd": 0.007918515641239979, "standard_error": 0.002799618053426266 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14524151292941784, 0.15793184280173087 ], "confidence_level": 0.95, "mean": 0.15212577320420634, "replicates": 8, "sample_sd": 0.009947080530794315, "standard_error": 0.003516824048166671 }, "log_loss": { "bootstrap_interval": [ 0.6165635945894955, 0.6728840853523564 ], "confidence_level": 0.95, "mean": 0.6457562078731067, "replicates": 8, "sample_sd": 0.043263947908581006, "standard_error": 0.01529611547352959 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19649564853962925, 0.20362721458419372 ], "confidence_level": 0.95, "mean": 0.19971191477641081, "replicates": 8, "sample_sd": 0.005506869817303598, "standard_error": 0.001946972495463449 }, "log_loss": { "bootstrap_interval": [ 0.5763251763296524, 0.5922519916553793 ], "confidence_level": 0.95, "mean": 0.5834449207252018, "replicates": 8, "sample_sd": 0.012400593424502377, "standard_error": 0.004384271850601471 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12800408969480137, 0.13735102405568075 ], "confidence_level": 0.95, "mean": 0.13305272058605566, "replicates": 8, "sample_sd": 0.007361285952366421, "standard_error": 0.0026026076075857845 }, "log_loss": { "bootstrap_interval": [ 0.40075802289438756, 0.4764867036845577 ], "confidence_level": 0.95, "mean": 0.4314834912358202, "replicates": 8, "sample_sd": 0.060782848481943055, "standard_error": 0.02148998217070819 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.12589468476014876, 0.13414015639038912 ], "confidence_level": 0.95, "mean": 0.13025210693597342, "replicates": 8, "sample_sd": 0.006470934224344471, "standard_error": 0.0022878207353230436 }, "log_loss": { "bootstrap_interval": [ 0.3933120062615949, 0.4151608694776497 ], "confidence_level": 0.95, "mean": 0.40447940240053437, "replicates": 8, "sample_sd": 0.01679389312322516, "standard_error": 0.005937537854977319 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3076923076923077 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3076923076923077, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.25-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04118562788958757, -0.03730089322877174 ], "confidence_level": 0.95, "mean": -0.03925075908720874, "replicates": 8, "sample_sd": 0.0030189123648006314, "standard_error": 0.0010673467024792213 }, "log_loss": { "bootstrap_interval": [ -0.16946993332701338, -0.1469886176567738 ], "confidence_level": 0.95, "mean": -0.15910391707493188, "replicates": 8, "sample_sd": 0.01726837312885611, "standard_error": 0.006105291869736857 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0033502235987224794, -0.0014771582906827725 ], "confidence_level": 0.95, "mean": -0.0024274206839263175, "replicates": 8, "sample_sd": 0.001438096384018669, "standard_error": 0.0005084438525697271 }, "log_loss": { "bootstrap_interval": [ -0.06701124242666189, -0.03573704458448258 ], "confidence_level": 0.95, "mean": -0.05295791878729567, "replicates": 8, "sample_sd": 0.024706906941802668, "standard_error": 0.008735210720346826 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0003257863408041963, 0.0027118773604322606 ], "confidence_level": 0.95, "mean": 0.0013693821789060867, "replicates": 8, "sample_sd": 0.0018537928863314652, "standard_error": 0.0006554147604201808 }, "log_loss": { "bootstrap_interval": [ 0.008765989176126322, 0.03036514749424509 ], "confidence_level": 0.95, "mean": 0.017581174674641847, "replicates": 8, "sample_sd": 0.017672578357796575, "standard_error": 0.006248199998924288 } } }, "discount": { "bootstrap_interval": [ 0.30939865722346693, 0.6520788597793101 ], "confidence_level": 0.95, "mean": 0.4703470248447459, "replicates": 8, "sample_sd": 0.2674851174733934, "standard_error": 0.09457027021595836 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.18689118124063325, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06563625437126266, 0.06889886819679357 ], "confidence_level": 0.95, "mean": 0.06736204645817191, "replicates": 8, "sample_sd": 0.002577394201855956, "standard_error": 0.000911246458961618 }, "log_loss": { "bootstrap_interval": [ 0.2651078625267914, 0.2742502633829907 ], "confidence_level": 0.95, "mean": 0.26981446126455166, "replicates": 8, "sample_sd": 0.0070841356637123525, "standard_error": 0.002504620183328234 } }, "independent": { "brier": { "bootstrap_interval": [ 0.02814165352889922, 0.03321152168998517 ], "confidence_level": 0.95, "mean": 0.030538708054889487, "replicates": 8, "sample_sd": 0.003923648567622755, "standard_error": 0.001387219254579467 }, "log_loss": { "bootstrap_interval": [ 0.15169649763118231, 0.1783594226982675 ], "confidence_level": 0.95, "mean": 0.16366846297691545, "replicates": 8, "sample_sd": 0.020741160968417074, "standard_error": 0.007333107785224726 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11072669563126027, 0.11676917568794884 ], "confidence_level": 0.95, "mean": 0.1136443487378146, "replicates": 8, "sample_sd": 0.004602628198623606, "standard_error": 0.0016272748052635878 }, "log_loss": { "bootstrap_interval": [ 0.3510516813878656, 0.36760874000959315 ], "confidence_level": 0.95, "mean": 0.3590003481495725, "replicates": 8, "sample_sd": 0.0128353060091697, "standard_error": 0.004537965958844168 } }, "learned": { "brier": { "bootstrap_interval": [ 0.026097159535835652, 0.03019254022576231 ], "confidence_level": 0.95, "mean": 0.02811128737096317, "replicates": 8, "sample_sd": 0.00314676250848504, "standard_error": 0.0011125485542666813 }, "log_loss": { "bootstrap_interval": [ 0.10046446411375595, 0.12213010443315295 ], "confidence_level": 0.95, "mean": 0.11071054418961977, "replicates": 8, "sample_sd": 0.016840490290097543, "standard_error": 0.00595401244131709 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.025259820595389202, 0.028336344940809905 ], "confidence_level": 0.95, "mean": 0.026741905192057082, "replicates": 8, "sample_sd": 0.0023498457714016406, "standard_error": 0.0008307959398503169 }, "log_loss": { "bootstrap_interval": [ 0.08842726465346108, 0.09834780456212287 ], "confidence_level": 0.95, "mean": 0.09312936951497792, "replicates": 8, "sample_sd": 0.007625586646294309, "standard_error": 0.002696052014060144 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3076923076923077 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3076923076923077, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.5-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.012081488244892687, -0.00800935706440914 ], "confidence_level": 0.95, "mean": -0.010185415148053586, "replicates": 8, "sample_sd": 0.0031198438098624564, "standard_error": 0.0011030313570983084 }, "log_loss": { "bootstrap_interval": [ -0.0364961313160797, -0.006587144128044576 ], "confidence_level": 0.95, "mean": -0.024357722474084256, "replicates": 8, "sample_sd": 0.0242381089453857, "standard_error": 0.008569465599210273 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.047009840409273376, -0.036127848752187455 ], "confidence_level": 0.95, "mean": -0.04201696294925095, "replicates": 8, "sample_sd": 0.008485141712595096, "standard_error": 0.0029999506221524137 }, "log_loss": { "bootstrap_interval": [ -0.6618908329950379, -0.5641165992645251 ], "confidence_level": 0.95, "mean": -0.6166384727121209, "replicates": 8, "sample_sd": 0.0762414649481509, "standard_error": 0.026955428436216983 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0008630286996847356, 0.005322406129953064 ], "confidence_level": 0.95, "mean": 0.0028024055123577994, "replicates": 8, "sample_sd": 0.003536865977452532, "standard_error": 0.0012504709584023358 }, "log_loss": { "bootstrap_interval": [ 0.003065915129455636, 0.03411600223972348 ], "confidence_level": 0.95, "mean": 0.015091952489907225, "replicates": 8, "sample_sd": 0.026033144053997044, "standard_error": 0.009204106348093778 } } }, "discount": { "bootstrap_interval": [ 0.15847823415632264, 0.2711974416691562 ], "confidence_level": 0.95, "mean": 0.20632189113768118, "replicates": 8, "sample_sd": 0.08883079183262085, "standard_error": 0.03140642764150839 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.4051001275842926, "discount_min": 0.11645658383720736, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1760920863758861, 0.18262557003794308 ], "confidence_level": 0.95, "mean": 0.17937682636867563, "replicates": 8, "sample_sd": 0.005090003223893175, "standard_error": 0.0017995878979381263 }, "log_loss": { "bootstrap_interval": [ 0.5332141726841515, 0.5475556964059222 ], "confidence_level": 0.95, "mean": 0.5404662696499132, "replicates": 8, "sample_sd": 0.010965913972710288, "standard_error": 0.0038770360660058787 } }, "independent": { "brier": { "bootstrap_interval": [ 0.20313231768047207, 0.2182412276153465 ], "confidence_level": 0.95, "mean": 0.21120837416987298, "replicates": 8, "sample_sd": 0.011801311860294933, "standard_error": 0.004172393821655888 }, "log_loss": { "bootstrap_interval": [ 1.0778640800559147, 1.177794288316602 ], "confidence_level": 0.95, "mean": 1.13274701988795, "replicates": 8, "sample_sd": 0.07771081976532881, "standard_error": 0.027474923813814795 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19538789402363907, 0.20356075558717276 ], "confidence_level": 0.95, "mean": 0.19920056907551684, "replicates": 8, "sample_sd": 0.00638457994517526, "standard_error": 0.002257289887130531 }, "log_loss": { "bootstrap_interval": [ 0.5736637713181861, 0.5916634489688349 ], "confidence_level": 0.95, "mean": 0.5821224059622203, "replicates": 8, "sample_sd": 0.01413800323726898, "standard_error": 0.0049985389807551284 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16436360110114118, 0.1736531292150041 ], "confidence_level": 0.95, "mean": 0.16919141122062203, "replicates": 8, "sample_sd": 0.007290430750197002, "standard_error": 0.0025775565106176145 }, "log_loss": { "bootstrap_interval": [ 0.49886670382805554, 0.5356295126087072 ], "confidence_level": 0.95, "mean": 0.5161085471758289, "replicates": 8, "sample_sd": 0.028443877256749392, "standard_error": 0.010056429245742654 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.16135796943443326, 0.17103915338778033 ], "confidence_level": 0.95, "mean": 0.16638900570826426, "replicates": 8, "sample_sd": 0.007518463549948486, "standard_error": 0.002658178280136228 }, "log_loss": { "bootstrap_interval": [ 0.48933969315076487, 0.5125047724652718 ], "confidence_level": 0.95, "mean": 0.5010165946859216, "replicates": 8, "sample_sd": 0.017863272609921405, "standard_error": 0.006315620598329671 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.18181818181818182 ], "oracle_method": "oracle", "population_optimal_scalar": 0.18181818181818182, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.5-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.018347894238607332, -0.011410269881403725 ], "confidence_level": 0.95, "mean": -0.015314812203868218, "replicates": 8, "sample_sd": 0.005533975042818047, "standard_error": 0.0019565556398468778 }, "log_loss": { "bootstrap_interval": [ -0.07812907627922647, 0.053370014781649454 ], "confidence_level": 0.95, "mean": -0.031336917058423404, "replicates": 8, "sample_sd": 0.1182084718512094, "standard_error": 0.04179300601984464 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.014767062069363363, -0.0067535110126989255 ], "confidence_level": 0.95, "mean": -0.010840808613011352, "replicates": 8, "sample_sd": 0.006129531232265706, "standard_error": 0.0021671165499149076 }, "log_loss": { "bootstrap_interval": [ -0.3803817447341526, -0.2231350412797346 ], "confidence_level": 0.95, "mean": -0.3189455436346246, "replicates": 8, "sample_sd": 0.1325432739854081, "standard_error": 0.04686112391787429 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00039509162153285315, 0.006269156100556733 ], "confidence_level": 0.95, "mean": 0.0026424634597323323, "replicates": 8, "sample_sd": 0.005023935886039808, "standard_error": 0.001776229566632597 }, "log_loss": { "bootstrap_interval": [ 0.0036208142768659854, 0.13523647362389532 ], "confidence_level": 0.95, "mean": 0.04923035748395894, "replicates": 8, "sample_sd": 0.120335087785966, "standard_error": 0.04254487829406752 } } }, "discount": { "bootstrap_interval": [ 0.1711382423797478, 0.5099238922650903 ], "confidence_level": 0.95, "mean": 0.29894200092217593, "replicates": 8, "sample_sd": 0.28793620027012773, "standard_error": 0.10180081988004756 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.11868053777960419, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07713987912054905, 0.08229490186570987 ], "confidence_level": 0.95, "mean": 0.07986507495714226, "replicates": 8, "sample_sd": 0.0039793306641505485, "standard_error": 0.0014069058486022102 }, "log_loss": { "bootstrap_interval": [ 0.2794793218904283, 0.2926426631298806 ], "confidence_level": 0.95, "mean": 0.28631217712288143, "replicates": 8, "sample_sd": 0.010178561861869254, "standard_error": 0.0035986650576272598 } }, "independent": { "brier": { "bootstrap_interval": [ 0.06959026902984687, 0.08077252720706687 ], "confidence_level": 0.95, "mean": 0.0753910713662854, "replicates": 8, "sample_sd": 0.00868965585216055, "standard_error": 0.003072257289620046 }, "log_loss": { "bootstrap_interval": [ 0.543024604630636, 0.6042941063825855 ], "confidence_level": 0.95, "mean": 0.5739208036990826, "replicates": 8, "sample_sd": 0.0469896915463235, "standard_error": 0.016613364769134765 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10879642723388795, 0.115862238514116 ], "confidence_level": 0.95, "mean": 0.11230572728599801, "replicates": 8, "sample_sd": 0.005498152457137541, "standard_error": 0.0019438904432197166 }, "log_loss": { "bootstrap_interval": [ 0.346533308697969, 0.3655469995292494 ], "confidence_level": 0.95, "mean": 0.3558041579520046, "replicates": 8, "sample_sd": 0.014597153738659536, "standard_error": 0.005160873197314361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.05995587982881137, 0.06866365469468164 ], "confidence_level": 0.95, "mean": 0.06455026275327404, "replicates": 8, "sample_sd": 0.006877267700787615, "standard_error": 0.002431481313631069 }, "log_loss": { "bootstrap_interval": [ 0.20551989420474887, 0.34058989424440617 ], "confidence_level": 0.95, "mean": 0.25497526006445803, "replicates": 8, "sample_sd": 0.11674741487070785, "standard_error": 0.041276444370538345 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.05831187961142288, 0.06492481144665613 ], "confidence_level": 0.95, "mean": 0.061907799293541704, "replicates": 8, "sample_sd": 0.005074695835775628, "standard_error": 0.0017941759189680404 }, "log_loss": { "bootstrap_interval": [ 0.19707929902593066, 0.21346154995843733 ], "confidence_level": 0.95, "mean": 0.2057449025804991, "replicates": 8, "sample_sd": 0.012748919266025088, "standard_error": 0.004507423632903081 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.18181818181818182 ], "oracle_method": "oracle", "population_optimal_scalar": 0.18181818181818182, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.75-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0013229932688278336, 0.0013274197985157499 ], "confidence_level": 0.95, "mean": 1.6590826263134656e-05, "replicates": 8, "sample_sd": 0.0020659844647131333, "standard_error": 0.000730435812412358 }, "log_loss": { "bootstrap_interval": [ -0.004418289184915644, 0.009661426091128383 ], "confidence_level": 0.95, "mean": 0.0020263023126374097, "replicates": 8, "sample_sd": 0.011069046126167438, "standard_error": 0.00391349878853984 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06783656412936218, -0.05349726497034643 ], "confidence_level": 0.95, "mean": -0.06138827054355004, "replicates": 8, "sample_sd": 0.011153501899051425, "standard_error": 0.003943358413398149 }, "log_loss": { "bootstrap_interval": [ -1.0729141639908841, -0.9406492007329468 ], "confidence_level": 0.95, "mean": -1.0137211747611035, "replicates": 8, "sample_sd": 0.10280203415162606, "standard_error": 0.036346007734192914 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010512643517295705, 0.004838586798412866 ], "confidence_level": 0.95, "mean": 0.0027818788797651095, "replicates": 8, "sample_sd": 0.0029043797037897963, "standard_error": 0.0010268532918451705 }, "log_loss": { "bootstrap_interval": [ 0.003581264264679057, 0.019469183684305592 ], "confidence_level": 0.95, "mean": 0.010731136320241808, "replicates": 8, "sample_sd": 0.012531162655616834, "standard_error": 0.004430435044969144 } } }, "discount": { "bootstrap_interval": [ 0.11201787015723583, 0.1765197022291448 ], "confidence_level": 0.95, "mean": 0.14190185899571373, "replicates": 8, "sample_sd": 0.04995407416781532, "standard_error": 0.017661432295978975 }, "discount_below_exchangeable_range_count": 2, "discount_max": 0.23969168707461108, "discount_min": 0.08674917014589184, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18432052663406445, 0.19246797377091454 ], "confidence_level": 0.95, "mean": 0.18852231007357234, "replicates": 8, "sample_sd": 0.006303068083112255, "standard_error": 0.0022284710919245844 }, "log_loss": { "bootstrap_interval": [ 0.5494288948718736, 0.5670100880508299 ], "confidence_level": 0.95, "mean": 0.5582727301927891, "replicates": 8, "sample_sd": 0.013666842926758436, "standard_error": 0.004831958655461145 } }, "independent": { "brier": { "bootstrap_interval": [ 0.23964678632205033, 0.25863342673420747 ], "confidence_level": 0.95, "mean": 0.2499271714433855, "replicates": 8, "sample_sd": 0.014925022438777993, "standard_error": 0.005276792287910651 }, "log_loss": { "bootstrap_interval": [ 1.4977980152186192, 1.6383695950894848 ], "confidence_level": 0.95, "mean": 1.57402020726653, "replicates": 8, "sample_sd": 0.10997982746133786, "standard_error": 0.03888374089581924 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19424103441151375, 0.20341219508417513 ], "confidence_level": 0.95, "mean": 0.19869529628463112, "replicates": 8, "sample_sd": 0.007071188646351022, "standard_error": 0.0025000427214420654 }, "log_loss": { "bootstrap_interval": [ 0.5712924540430874, 0.5912517894068656 ], "confidence_level": 0.95, "mean": 0.58080379894007, "replicates": 8, "sample_sd": 0.015418508102089555, "standard_error": 0.005451265817383624 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18427317060478524, 0.19288649450365858 ], "confidence_level": 0.95, "mean": 0.18853890089983547, "replicates": 8, "sample_sd": 0.006758962094710005, "standard_error": 0.002389653965476138 }, "log_loss": { "bootstrap_interval": [ 0.5487216132904054, 0.5722899099720854 ], "confidence_level": 0.95, "mean": 0.5602990325054266, "replicates": 8, "sample_sd": 0.01844704182084483, "standard_error": 0.0065220141821756075 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1805683059995467, 0.19054756795096803 ], "confidence_level": 0.95, "mean": 0.18575702202007036, "replicates": 8, "sample_sd": 0.007629841780704515, "standard_error": 0.0026975564312583028 }, "log_loss": { "bootstrap_interval": [ 0.5377339912544388, 0.5602784651000307 ], "confidence_level": 0.95, "mean": 0.5495678961851848, "replicates": 8, "sample_sd": 0.017269223111819238, "standard_error": 0.006105592384095417 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12903225806451613 ], "oracle_method": "oracle", "population_optimal_scalar": 0.12903225806451613, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho0.75-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.004808510328688374, -0.00048591700750474806 ], "confidence_level": 0.95, "mean": -0.002893480916336403, "replicates": 8, "sample_sd": 0.003402809988403875, "standard_error": 0.0012030750089448484 }, "log_loss": { "bootstrap_interval": [ -0.020159548298309068, 0.01965938487744548 ], "confidence_level": 0.95, "mean": -0.004788911675017868, "replicates": 8, "sample_sd": 0.033634013420257104, "standard_error": 0.011891419483991571 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.024393381204523407, -0.016866896711776887 ], "confidence_level": 0.95, "mean": -0.020716047373037456, "replicates": 8, "sample_sd": 0.005793076615435099, "standard_error": 0.002048161879353686 }, "log_loss": { "bootstrap_interval": [ -0.8148781164483181, -0.6997498476527239 ], "confidence_level": 0.95, "mean": -0.7606269627761926, "replicates": 8, "sample_sd": 0.08891478164133297, "standard_error": 0.031436122523153844 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0004870862276330753, 0.004237303108163044 ], "confidence_level": 0.95, "mean": 0.001967584119667543, "replicates": 8, "sample_sd": 0.0031000315145172856, "standard_error": 0.001096026652903588 }, "log_loss": { "bootstrap_interval": [ 0.003062358586833418, 0.04263082573454137 ], "confidence_level": 0.95, "mean": 0.017583258574267867, "replicates": 8, "sample_sd": 0.0346571014269304, "standard_error": 0.01225313571762623 } } }, "discount": { "bootstrap_interval": [ 0.11818083574355268, 0.21141315646207495 ], "confidence_level": 0.95, "mean": 0.1581056755143028, "replicates": 8, "sample_sd": 0.07369240087698098, "standard_error": 0.026054198191015364 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.32570492946900365, "discount_min": 0.08716857324663815, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09003077661085034, 0.09709691526756423 ], "confidence_level": 0.95, "mean": 0.09384373226753306, "replicates": 8, "sample_sd": 0.005541636721665967, "standard_error": 0.001959264452381197 }, "log_loss": { "bootstrap_interval": [ 0.30181631808961734, 0.31884898113315707 ], "confidence_level": 0.95, "mean": 0.3108607034770697, "replicates": 8, "sample_sd": 0.013436054523369012, "standard_error": 0.004750362632933207 } }, "independent": { "brier": { "bootstrap_interval": [ 0.10373959036665513, 0.11842712777076421 ], "confidence_level": 0.95, "mean": 0.11166629872423413, "replicates": 8, "sample_sd": 0.011460320122073967, "standard_error": 0.004051835036443571 }, "log_loss": { "bootstrap_interval": [ 1.0028228510048258, 1.1218167309901559 ], "confidence_level": 0.95, "mean": 1.0666987545782445, "replicates": 8, "sample_sd": 0.09111697561395822, "standard_error": 0.03221471566891957 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10697188025842762, 0.11529500612305989 ], "confidence_level": 0.95, "mean": 0.11121122109872107, "replicates": 8, "sample_sd": 0.006408123641965544, "standard_error": 0.002265613840957836 }, "log_loss": { "bootstrap_interval": [ 0.34245507124171415, 0.3629722217186941 ], "confidence_level": 0.95, "mean": 0.3526925037658831, "replicates": 8, "sample_sd": 0.01593143496684411, "standard_error": 0.005632612849543975 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08603498436022738, 0.09490288613822558 ], "confidence_level": 0.95, "mean": 0.09095025135119666, "replicates": 8, "sample_sd": 0.006988793071613687, "standard_error": 0.002470911486623799 }, "log_loss": { "bootstrap_interval": [ 0.2861611489496011, 0.3312948144320727 ], "confidence_level": 0.95, "mean": 0.3060717918020518, "replicates": 8, "sample_sd": 0.03496839952318773, "standard_error": 0.01236319621504324 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08443142430433764, 0.09270991612124621 ], "confidence_level": 0.95, "mean": 0.08898266723152912, "replicates": 8, "sample_sd": 0.006543706337425765, "standard_error": 0.002313549562643572 }, "log_loss": { "bootstrap_interval": [ 0.27707656151031235, 0.2980040208973086 ], "confidence_level": 0.95, "mean": 0.28848853322778395, "replicates": 8, "sample_sd": 0.016212037980390764, "standard_error": 0.005731820996394085 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12903225806451613 ], "oracle_method": "oracle", "population_optimal_scalar": 0.12903225806451613, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho1-a0.5": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0010529360244109186, 0.005107377907497923 ], "confidence_level": 0.95, "mean": 0.0026696714486234216, "replicates": 8, "sample_sd": 0.0032450327229714957, "standard_error": 0.0011472923217926958 }, "log_loss": { "bootstrap_interval": [ 0.0036102167277028228, 0.014126772108320668 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.08256310642731206, -0.06701881322266519 ], "confidence_level": 0.95, "mean": -0.0756835221532101, "replicates": 8, "sample_sd": 0.012127076345971752, "standard_error": 0.004287568960101801 }, "log_loss": { "bootstrap_interval": [ -1.469388295841948, -1.3054321335002865 ], "confidence_level": 0.95, "mean": -1.3940027171482738, "replicates": 8, "sample_sd": 0.1285141122574556, "standard_error": 0.045436600127708025 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010529360244109186, 0.005107377907497923 ], "confidence_level": 0.95, "mean": 0.0026696714486234216, "replicates": 8, "sample_sd": 0.0032450327229714957, "standard_error": 0.0011472923217926958 }, "log_loss": { "bootstrap_interval": [ 0.0036102167277028228, 0.014126772108320668 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } } }, "discount": { "bootstrap_interval": [ 0.08711947897957696, 0.12752996809166173 ], "confidence_level": 0.95, "mean": 0.10769208585735221, "replicates": 8, "sample_sd": 0.0315341069275239, "standard_error": 0.011148990423556916 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.15184755094770258, "discount_min": 0.061494310158395404, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1930308007194194, 0.2026638326969519 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5684267353075054, 0.589351981876395 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "independent": { "brier": { "bootstrap_interval": [ 0.26564571517686797, 0.28575807262105596 ], "confidence_level": 0.95, "mean": 0.2764367573866667, "replicates": 8, "sample_sd": 0.015721556751720014, "standard_error": 0.005558409694975186 }, "log_loss": { "bootstrap_interval": [ 1.887864797336853, 2.0626979638471368 ], "confidence_level": 0.95, "mean": 1.9814877631513514, "replicates": 8, "sample_sd": 0.1361690890782133, "standard_error": 0.04814304313759983 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19312038671175585, 0.20263042521373462 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5684267353075054, 0.589495229462119 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19651757790393537, 0.2048896922172256 ], "confidence_level": 0.95, "mean": 0.20075323523345656, "replicates": 8, "sample_sd": 0.006395098492831272, "standard_error": 0.002261008755318431 }, "log_loss": { "bootstrap_interval": [ 0.577673102149387, 0.5972077444817651 ], "confidence_level": 0.95, "mean": 0.5874850460030775, "replicates": 8, "sample_sd": 0.015094497522713133, "standard_error": 0.005336710778456999 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19299740962567577, 0.20263111724479335 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.568546415598249, 0.5894489306743499 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1 ], "oracle_method": "oracle", "population_optimal_scalar": 0.1, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n10-rho1-a1": { "condition": { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.000796217994750203, 0.0034158732743658233 ], "confidence_level": 0.95, "mean": 0.0020336014594108456, "replicates": 8, "sample_sd": 0.00203507141594228, "standard_error": 0.0007195063992058476 }, "log_loss": { "bootstrap_interval": [ 0.0046906699396142065, 0.020085568467703686 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922939, "standard_error": 0.004312314006328675 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.034942564206805984, -0.026134941859041284 ], "confidence_level": 0.95, "mean": -0.030744201308687918, "replicates": 8, "sample_sd": 0.0068048141683774415, "standard_error": 0.0024058651215869927 }, "log_loss": { "bootstrap_interval": [ -1.3082765366479163, -1.1477526768663262 ], "confidence_level": 0.95, "mean": -1.2337353838218332, "replicates": 8, "sample_sd": 0.12592361601338958, "standard_error": 0.044520721397299345 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.000796217994750203, 0.0034158732743658233 ], "confidence_level": 0.95, "mean": 0.002033601459410844, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.0046906699396142065, 0.020085568467703686 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922939, "standard_error": 0.004312314006328675 } } }, "discount": { "bootstrap_interval": [ 0.09040457854200867, 0.1379946713845034 ], "confidence_level": 0.95, "mean": 0.11298110643835674, "replicates": 8, "sample_sd": 0.03664122460406781, "standard_error": 0.012954629194257859 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.18121410578456298, "discount_min": 0.07051174186054293, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 10 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1047745116569177, 0.11424900267635442 ], "confidence_level": 0.95, "mean": 0.10991164242283745, "replicates": 8, "sample_sd": 0.007392589799104922, "standard_error": 0.0026136751887387935 }, "log_loss": { "bootstrap_interval": [ 0.33687978914709854, 0.360204302947993 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1335131248994869, 0.15049015722542913 ], "confidence_level": 0.95, "mean": 0.1426894451909362, "replicates": 8, "sample_sd": 0.013249038099547888, "standard_error": 0.00468424234219462 }, "log_loss": { "bootstrap_interval": [ 1.4975459925502188, 1.6756558450572916 ], "confidence_level": 0.95, "mean": 1.5946064629992507, "replicates": 8, "sample_sd": 0.13789584044140854, "standard_error": 0.04875354193676907 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10489646074140024, 0.11424900267635443 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.3367588877952547, 0.35990000757315765 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10736093769403166, 0.11596952363855859 ], "confidence_level": 0.95, "mean": 0.1119452438822483, "replicates": 8, "sample_sd": 0.006750192655061165, "standard_error": 0.002386553500354688 }, "log_loss": { "bootstrap_interval": [ 0.3488529360460216, 0.37178070455972473 ], "confidence_level": 0.95, "mean": 0.3608710791774176, "replicates": 8, "sample_sd": 0.01801647685394999, "standard_error": 0.0063697864782592565 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10488710309379178, 0.11420151204307634 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.3368229045995601, 0.36030796120479147 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1 ], "oracle_method": "oracle", "population_optimal_scalar": 0.1, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.015600128213093412, -0.012728724524799182 ], "confidence_level": 0.95, "mean": -0.01414814527415107, "replicates": 8, "sample_sd": 0.002200467666251775, "standard_error": 0.0007779828042941833 }, "log_loss": { "bootstrap_interval": [ -0.04684604576916468, -0.03756896013687132 ], "confidence_level": 0.95, "mean": -0.042225671414832425, "replicates": 8, "sample_sd": 0.007199039410336918, "standard_error": 0.0025452447925392195 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0001474327073067265, 0.0019364639706487899 ], "confidence_level": 0.95, "mean": 0.0005963437544473545, "replicates": 8, "sample_sd": 0.0018506935650996392, "standard_error": 0.000654318984890131 }, "log_loss": { "bootstrap_interval": [ -0.00037151036651394964, 0.006964734846300627 ], "confidence_level": 0.95, "mean": 0.002197741493262226, "replicates": 8, "sample_sd": 0.006625528333892088, "standard_error": 0.0023424780069193516 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.0001474327073067265, 0.0019364639706487899 ], "confidence_level": 0.95, "mean": 0.0005963437544473545, "replicates": 8, "sample_sd": 0.0018506935650996392, "standard_error": 0.000654318984890131 }, "log_loss": { "bootstrap_interval": [ -0.00037151036651394964, 0.006964734846300627 ], "confidence_level": 0.95, "mean": 0.002197741493262226, "replicates": 8, "sample_sd": 0.006625528333892088, "standard_error": 0.0023424780069193516 } } }, "discount": { "bootstrap_interval": [ 0.8630145826659902, 1.0 ], "confidence_level": 0.95, "mean": 0.9432398183625123, "replicates": 8, "sample_sd": 0.11646012049028584, "standard_error": 0.04117487046824175 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.679099057213912, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1777703239861381, 0.18271944704878698 ], "confidence_level": 0.95, "mean": 0.18003477041390883, "replicates": 8, "sample_sd": 0.0037763241237447555, "standard_error": 0.0013351321979291318 }, "log_loss": { "bootstrap_interval": [ 0.5373351591261742, 0.5481269237351161 ], "confidence_level": 0.95, "mean": 0.542320337767303, "replicates": 8, "sample_sd": 0.008340840752176905, "standard_error": 0.002948932528330696 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1615994192262355, 0.16939343831823023 ], "confidence_level": 0.95, "mean": 0.1652902813853104, "replicates": 8, "sample_sd": 0.006077480433776236, "standard_error": 0.002148713813625868 }, "log_loss": { "bootstrap_interval": [ 0.4880803479903896, 0.5090456628155162 ], "confidence_level": 0.95, "mean": 0.49789692485920833, "replicates": 8, "sample_sd": 0.01605935029741968, "standard_error": 0.0056778377483778265 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1976170632399488, 0.20337260671580593 ], "confidence_level": 0.95, "mean": 0.200401419084981, "replicates": 8, "sample_sd": 0.004402965745277611, "standard_error": 0.00155668346790894 }, "log_loss": { "bootstrap_interval": [ 0.5786455221369766, 0.5918658939460962 ], "confidence_level": 0.95, "mean": 0.5850450001139459, "replicates": 8, "sample_sd": 0.01030599781392939, "standard_error": 0.0036437204705616027 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1626221032466008, 0.16959283905555195 ], "confidence_level": 0.95, "mean": 0.16588662513975777, "replicates": 8, "sample_sd": 0.005432953439121209, "standard_error": 0.0019208391093366907 }, "log_loss": { "bootstrap_interval": [ 0.491260265772963, 0.509862808225227 ], "confidence_level": 0.95, "mean": 0.5000946663524706, "replicates": 8, "sample_sd": 0.014332845640081244, "standard_error": 0.005067426172900745 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.16158341549827326, 0.16935332350534776 ], "confidence_level": 0.95, "mean": 0.1652902813853104, "replicates": 8, "sample_sd": 0.006077480433776236, "standard_error": 0.002148713813625868 }, "log_loss": { "bootstrap_interval": [ 0.4881083244115013, 0.5087830235190091 ], "confidence_level": 0.95, "mean": 0.49789692485920833, "replicates": 8, "sample_sd": 0.01605935029741968, "standard_error": 0.0056778377483778265 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.02492377993282974, -0.021951410556742317 ], "confidence_level": 0.95, "mean": -0.023345923820579755, "replicates": 8, "sample_sd": 0.0023134809837493287, "standard_error": 0.0008179390458776376 }, "log_loss": { "bootstrap_interval": [ -0.0976442296361755, -0.08797351764583877 ], "confidence_level": 0.95, "mean": -0.09292798105629256, "replicates": 8, "sample_sd": 0.007386060107827241, "standard_error": 0.002611366594248042 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -9.067073994727462e-05, 0.0008914413242243008 ], "confidence_level": 0.95, "mean": 0.0003352710708028452, "replicates": 8, "sample_sd": 0.0007937747182472655, "standard_error": 0.0002806417430035413 }, "log_loss": { "bootstrap_interval": [ -0.00031513758676747494, 0.004773450370920727 ], "confidence_level": 0.95, "mean": 0.001823261929482834, "replicates": 8, "sample_sd": 0.004168604532594715, "standard_error": 0.0014738242665413505 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -9.067073994727462e-05, 0.0008914413242243008 ], "confidence_level": 0.95, "mean": 0.0003352710708028452, "replicates": 8, "sample_sd": 0.0007937747182472655, "standard_error": 0.0002806417430035413 }, "log_loss": { "bootstrap_interval": [ -0.00031513758676747494, 0.004773450370920727 ], "confidence_level": 0.95, "mean": 0.001823261929482834, "replicates": 8, "sample_sd": 0.004168604532594715, "standard_error": 0.0014738242665413505 } } }, "discount": { "bootstrap_interval": [ 0.8691230482825977, 0.9924829682454638 ], "confidence_level": 0.95, "mean": 0.9381176202378683, "replicates": 8, "sample_sd": 0.09853535835224683, "standard_error": 0.03483751003876012 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7683142235833402, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0808933287360985, 0.08556920111884804 ], "confidence_level": 0.95, "mean": 0.08298954770292179, "replicates": 8, "sample_sd": 0.003669316240023232, "standard_error": 0.0012972991978191762 }, "log_loss": { "bootstrap_interval": [ 0.2867198262812826, 0.297975881437086 ], "confidence_level": 0.95, "mean": 0.29166424700342314, "replicates": 8, "sample_sd": 0.008800461730771664, "standard_error": 0.003111433083700672 } }, "independent": { "brier": { "bootstrap_interval": [ 0.055457476040899026, 0.06328840776664584 ], "confidence_level": 0.95, "mean": 0.05930835281153918, "replicates": 8, "sample_sd": 0.005920179498865836, "standard_error": 0.0020930995347448046 }, "log_loss": { "bootstrap_interval": [ 0.18695674971675283, 0.20781428483418038 ], "confidence_level": 0.95, "mean": 0.19691300401764777, "replicates": 8, "sample_sd": 0.01607813319098356, "standard_error": 0.005684478504082489 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11210830881460178, 0.11753177801717457 ], "confidence_level": 0.95, "mean": 0.11474074946836885, "replicates": 8, "sample_sd": 0.004152437295533535, "standard_error": 0.0014681082850618453 }, "log_loss": { "bootstrap_interval": [ 0.35513316817102647, 0.3700992494736096 ], "confidence_level": 0.95, "mean": 0.36231777974411405, "replicates": 8, "sample_sd": 0.011648816374946308, "standard_error": 0.004118478525760715 } }, "learned": { "brier": { "bootstrap_interval": [ 0.05602878284240281, 0.06327412865880014 ], "confidence_level": 0.95, "mean": 0.059643623882342026, "replicates": 8, "sample_sd": 0.005591871831492678, "standard_error": 0.0019770252457872554 }, "log_loss": { "bootstrap_interval": [ 0.18967239903028507, 0.20899107570767902 ], "confidence_level": 0.95, "mean": 0.1987362659471306, "replicates": 8, "sample_sd": 0.014903395793718448, "standard_error": 0.005269146114222692 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.05554560274214265, 0.06315539348882522 ], "confidence_level": 0.95, "mean": 0.05930835281153918, "replicates": 8, "sample_sd": 0.005920179498865836, "standard_error": 0.0020930995347448046 }, "log_loss": { "bootstrap_interval": [ 0.1870011630391213, 0.2077130537387553 ], "confidence_level": 0.95, "mean": 0.19691300401764777, "replicates": 8, "sample_sd": 0.01607813319098356, "standard_error": 0.005684478504082489 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.1-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.012170005786681455, -0.008664298394923307 ], "confidence_level": 0.95, "mean": -0.010358126427778034, "replicates": 8, "sample_sd": 0.002702119731400307, "standard_error": 0.0009553435928255646 }, "log_loss": { "bootstrap_interval": [ -0.035983281148174076, -0.025123533433281327 ], "confidence_level": 0.95, "mean": -0.030661047602664408, "replicates": 8, "sample_sd": 0.00839597106334161, "standard_error": 0.00296842403676744 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0003574936003804749, 0.002973440940326831 ], "confidence_level": 0.95, "mean": 0.0008834361326399594, "replicates": 8, "sample_sd": 0.0028780253224391455, "standard_error": 0.0010175356109616597 }, "log_loss": { "bootstrap_interval": [ -0.0010994165083874352, 0.010825756957242465 ], "confidence_level": 0.95, "mean": 0.003347729211489507, "replicates": 8, "sample_sd": 0.01016490092257526, "standard_error": 0.0035938351862211796 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00017481234961088944, 0.00324337666445779 ], "confidence_level": 0.95, "mean": 0.001262936027299967, "replicates": 8, "sample_sd": 0.002744143564963769, "standard_error": 0.0009702012616676541 }, "log_loss": { "bootstrap_interval": [ 0.000698674095226115, 0.011441289579755931 ], "confidence_level": 0.95, "mean": 0.004576726180355081, "replicates": 8, "sample_sd": 0.009411940031424612, "standard_error": 0.003327623310170735 } } }, "discount": { "bootstrap_interval": [ 0.7590014926640309, 0.9669295551045914 ], "confidence_level": 0.95, "mean": 0.8738843129546937, "replicates": 8, "sample_sd": 0.1643629946520257, "standard_error": 0.0581110940472878 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.5528321097141686, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17890912832061256, 0.18456174924177268 ], "confidence_level": 0.95, "mean": 0.1816437457016079, "replicates": 8, "sample_sd": 0.004311422154102862, "standard_error": 0.0015243179208620228 }, "log_loss": { "bootstrap_interval": [ 0.5394373801231682, 0.5516032452958815 ], "confidence_level": 0.95, "mean": 0.5451871732159913, "replicates": 8, "sample_sd": 0.0094039238974941, "standard_error": 0.0033247891788401526 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1660325546191213, 0.17492508625915165 ], "confidence_level": 0.95, "mean": 0.17040218314118993, "replicates": 8, "sample_sd": 0.00680422751656715, "standard_error": 0.0024056577088503663 }, "log_loss": { "bootstrap_interval": [ 0.5000737090539398, 0.5230396402788966 ], "confidence_level": 0.95, "mean": 0.5111783964018374, "replicates": 8, "sample_sd": 0.01796065184685662, "standard_error": 0.006350049357721502 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19723085615431174, 0.20335671837431668 ], "confidence_level": 0.95, "mean": 0.20005045972324384, "replicates": 8, "sample_sd": 0.004854666888269692, "standard_error": 0.0017163839385486471 }, "log_loss": { "bootstrap_interval": [ 0.5779500314028742, 0.5918963737352921 ], "confidence_level": 0.95, "mean": 0.5842897509669385, "replicates": 8, "sample_sd": 0.01106838076050973, "standard_error": 0.003913263546255573 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16780019044555525, 0.17495851952769093 ], "confidence_level": 0.95, "mean": 0.17128561927382988, "replicates": 8, "sample_sd": 0.005496665696793945, "standard_error": 0.0019433647940592389 }, "log_loss": { "bootstrap_interval": [ 0.5056583674806058, 0.5234964325435597 ], "confidence_level": 0.95, "mean": 0.5145261256133269, "replicates": 8, "sample_sd": 0.013982842698404684, "standard_error": 0.0049436814461533765 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.16597144903334146, 0.1742793257632786 ], "confidence_level": 0.95, "mean": 0.17002268324652992, "replicates": 8, "sample_sd": 0.006515523422244082, "standard_error": 0.0023035853974242857 }, "log_loss": { "bootstrap_interval": [ 0.49944750777842817, 0.5210474020997544 ], "confidence_level": 0.95, "mean": 0.5099493994329718, "replicates": 8, "sample_sd": 0.016664689832296867, "standard_error": 0.005891857593393812 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.9090909090909091 ], "oracle_method": "oracle", "population_optimal_scalar": 0.9090909090909091, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.1-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.02028510116328914, -0.01790001913950677 ], "confidence_level": 0.95, "mean": -0.01903410609061629, "replicates": 8, "sample_sd": 0.0018396289799464149, "standard_error": 0.0006504070632937006 }, "log_loss": { "bootstrap_interval": [ -0.08135182059185654, -0.07169311680480109 ], "confidence_level": 0.95, "mean": -0.07639674862295509, "replicates": 8, "sample_sd": 0.007392662176000988, "standard_error": 0.002613700777835798 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 1.1312608220202802e-05, 0.0006773629355156137 ], "confidence_level": 0.95, "mean": 0.00030180395810821235, "replicates": 8, "sample_sd": 0.0005067302954793763, "standard_error": 0.00017915621408306492 }, "log_loss": { "bootstrap_interval": [ -0.00034888281401639423, 0.004628217745294443 ], "confidence_level": 0.95, "mean": 0.0018151519329406458, "replicates": 8, "sample_sd": 0.004010799962484376, "standard_error": 0.0014180319257277262 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00011503078983348802, 0.000791807184765075 ], "confidence_level": 0.95, "mean": 0.0004173501315049956, "replicates": 8, "sample_sd": 0.0005199103932284011, "standard_error": 0.00018381608233058343 }, "log_loss": { "bootstrap_interval": [ 0.0004766687058252287, 0.005242904973706487 ], "confidence_level": 0.95, "mean": 0.002515086386647443, "replicates": 8, "sample_sd": 0.003820580533472538, "standard_error": 0.0013507792016438744 } } }, "discount": { "bootstrap_interval": [ 0.8138099070859909, 0.9726833454947478 ], "confidence_level": 0.95, "mean": 0.9007226021772354, "replicates": 8, "sample_sd": 0.12235684498464974, "standard_error": 0.04325967740661852 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.6691946571716263, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08233148143007578, 0.08702358043320693 ], "confidence_level": 0.95, "mean": 0.08470544163847425, "replicates": 8, "sample_sd": 0.003673104922850452, "standard_error": 0.0012986386994786223 }, "log_loss": { "bootstrap_interval": [ 0.28817207610397183, 0.2999604487712564 ], "confidence_level": 0.95, "mean": 0.29421806927505517, "replicates": 8, "sample_sd": 0.009037043637026253, "standard_error": 0.003195077418810002 } }, "independent": { "brier": { "bootstrap_interval": [ 0.06190379487322337, 0.06855469652913704 ], "confidence_level": 0.95, "mean": 0.06536953158974974, "replicates": 8, "sample_sd": 0.005145015591729476, "standard_error": 0.001819037707111215 }, "log_loss": { "bootstrap_interval": [ 0.20571783170496147, 0.22573078337462435 ], "confidence_level": 0.95, "mean": 0.21600616871915945, "replicates": 8, "sample_sd": 0.015499271909777597, "standard_error": 0.005479820135428954 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11173952929335622, 0.11722595419149502 ], "confidence_level": 0.95, "mean": 0.11440040546560537, "replicates": 8, "sample_sd": 0.004214884081710997, "standard_error": 0.0014901865580465399 }, "log_loss": { "bootstrap_interval": [ 0.35349006111432263, 0.3686529492085284 ], "confidence_level": 0.95, "mean": 0.3609438993647717, "replicates": 8, "sample_sd": 0.011549219867090432, "standard_error": 0.00408326584271702 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06226550829948812, 0.06877412113931225 ], "confidence_level": 0.95, "mean": 0.06567133554785795, "replicates": 8, "sample_sd": 0.005100240344962268, "standard_error": 0.0018032072668020178 }, "log_loss": { "bootstrap_interval": [ 0.20786397885520325, 0.22748639141612342 ], "confidence_level": 0.95, "mean": 0.21782132065210008, "replicates": 8, "sample_sd": 0.015265615477321857, "standard_error": 0.0053972101115003 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06192715988070741, 0.06828243717980959 ], "confidence_level": 0.95, "mean": 0.06525398541635297, "replicates": 8, "sample_sd": 0.004943165929455068, "standard_error": 0.0017476730746239905 }, "log_loss": { "bootstrap_interval": [ 0.20590904338185712, 0.22455438393719743 ], "confidence_level": 0.95, "mean": 0.21530623426545265, "replicates": 8, "sample_sd": 0.01451890474501053, "standard_error": 0.0051332080002992435 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.9090909090909091 ], "oracle_method": "oracle", "population_optimal_scalar": 0.9090909090909091, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.25-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.00653011528520904, -0.003421534384174365 ], "confidence_level": 0.95, "mean": -0.005148857168335453, "replicates": 8, "sample_sd": 0.0024592945084457325, "standard_error": 0.0008694919119284072 }, "log_loss": { "bootstrap_interval": [ -0.01890675582247594, -0.009757776935238965 ], "confidence_level": 0.95, "mean": -0.014727241949849687, "replicates": 8, "sample_sd": 0.007216920929300845, "standard_error": 0.0025515668641978738 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0008026321340338857, 0.0035071591379704954 ], "confidence_level": 0.95, "mean": 0.0008926973482957408, "replicates": 8, "sample_sd": 0.003563157969940111, "standard_error": 0.0012597665814917723 }, "log_loss": { "bootstrap_interval": [ -0.0029697705915989886, 0.011420534735530813 ], "confidence_level": 0.95, "mean": 0.003022802360754931, "replicates": 8, "sample_sd": 0.011738177879747602, "standard_error": 0.00415007258877173 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010122064074912643, 0.004762502333978484 ], "confidence_level": 0.95, "mean": 0.002447831830155376, "replicates": 8, "sample_sd": 0.0031477988882267867, "standard_error": 0.001112914969838318 }, "log_loss": { "bootstrap_interval": [ 0.004073604625162236, 0.015741273765589953 ], "confidence_level": 0.95, "mean": 0.008682458551376879, "replicates": 8, "sample_sd": 0.009622061248407551, "standard_error": 0.003401912378870638 } } }, "discount": { "bootstrap_interval": [ 0.6670183905047776, 0.9499878402746522 ], "confidence_level": 0.95, "mean": 0.8122481026864694, "replicates": 8, "sample_sd": 0.221204376183159, "standard_error": 0.07820755721362588 }, "discount_below_exchangeable_range_count": 1, "discount_max": 1.0, "discount_min": 0.4631297673412472, "discount_one_count": 4, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18122344735770063, 0.18755320344666257 ], "confidence_level": 0.95, "mean": 0.1843394362127065, "replicates": 8, "sample_sd": 0.004899785084643701, "standard_error": 0.0017323356298541313 }, "log_loss": { "bootstrap_interval": [ 0.5433333033324482, 0.5573571117774552 ], "confidence_level": 0.95, "mean": 0.5501592116699349, "replicates": 8, "sample_sd": 0.010746317395579274, "standard_error": 0.0037993969515985313 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1737645534030514, 0.18300340556647687 ], "confidence_level": 0.95, "mean": 0.1782978816960753, "replicates": 8, "sample_sd": 0.007207814359590912, "standard_error": 0.002548347205600253 }, "log_loss": { "bootstrap_interval": [ 0.5201425057517423, 0.5453055108763144 ], "confidence_level": 0.95, "mean": 0.5324091673593302, "replicates": 8, "sample_sd": 0.01947112812334869, "standard_error": 0.006884083366685977 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19648545473237888, 0.2037380932351123 ], "confidence_level": 0.95, "mean": 0.19971191477641081, "replicates": 8, "sample_sd": 0.005506869817303598, "standard_error": 0.001946972495463449 }, "log_loss": { "bootstrap_interval": [ 0.5762357731497377, 0.5920290958680465 ], "confidence_level": 0.95, "mean": 0.5834449207252018, "replicates": 8, "sample_sd": 0.012400593424502377, "standard_error": 0.004384271850601471 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17622960881818758, 0.18250092393595102 ], "confidence_level": 0.95, "mean": 0.17919057904437105, "replicates": 8, "sample_sd": 0.004943220648419525, "standard_error": 0.001747692420699404 }, "log_loss": { "bootstrap_interval": [ 0.5279408911071541, 0.5435394584955963 ], "confidence_level": 0.95, "mean": 0.5354319697200851, "replicates": 8, "sample_sd": 0.012042132572965469, "standard_error": 0.004257536801145645 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1724938452588576, 0.18114211223816368 ], "confidence_level": 0.95, "mean": 0.17674274721421568, "replicates": 8, "sample_sd": 0.0066397181148160345, "standard_error": 0.002347494852076789 }, "log_loss": { "bootstrap_interval": [ 0.5163304179503347, 0.5380100821195452 ], "confidence_level": 0.95, "mean": 0.5267495111687083, "replicates": 8, "sample_sd": 0.016620068011752327, "standard_error": 0.0058760813974458445 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.8 ], "oracle_method": "oracle", "population_optimal_scalar": 0.8, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.25-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.014551125093783646, -0.011398799801076845 ], "confidence_level": 0.95, "mean": -0.013057222760618149, "replicates": 8, "sample_sd": 0.0024515264869098934, "standard_error": 0.0008667455015762097 }, "log_loss": { "bootstrap_interval": [ -0.055847154187935955, -0.04527755186658915 ], "confidence_level": 0.95, "mean": -0.051252979283434356, "replicates": 8, "sample_sd": 0.008236660612191093, "standard_error": 0.0029120992866062305 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -4.762282957240273e-05, 0.0008919653405862922 ], "confidence_level": 0.95, "mean": 0.00038176171591574985, "replicates": 8, "sample_sd": 0.0007443971887521854, "standard_error": 0.0002631841500314363 }, "log_loss": { "bootstrap_interval": [ -0.00222882628367169, 0.006472521974064429 ], "confidence_level": 0.95, "mean": 0.001640138618925216, "replicates": 8, "sample_sd": 0.006643446095009719, "standard_error": 0.0023488128921143304 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00031366415567700824, 0.0016175115797073328 ], "confidence_level": 0.95, "mean": 0.0009315248601453139, "replicates": 8, "sample_sd": 0.0010149553688274054, "standard_error": 0.00035884091194977587 }, "log_loss": { "bootstrap_interval": [ 0.0026532705272004327, 0.01073152601625105 ], "confidence_level": 0.95, "mean": 0.005984372702301854, "replicates": 8, "sample_sd": 0.006394985905804208, "standard_error": 0.0022609689497932756 } } }, "discount": { "bootstrap_interval": [ 0.7147029831022955, 0.9456624531631722 ], "confidence_level": 0.95, "mean": 0.8341372398597231, "replicates": 8, "sample_sd": 0.1766452366762405, "standard_error": 0.06245352235903614 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.528890494268663, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08540983771730687, 0.09030118283531775 ], "confidence_level": 0.95, "mean": 0.08797376516180504, "replicates": 8, "sample_sd": 0.003777874586753967, "standard_error": 0.001335680369383028 }, "log_loss": { "bootstrap_interval": [ 0.2932266440285334, 0.3059232396771397 ], "confidence_level": 0.95, "mean": 0.29981241321637403, "replicates": 8, "sample_sd": 0.009808590620913054, "standard_error": 0.0034678604709651945 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07092773072614311, 0.07780356975414221 ], "confidence_level": 0.95, "mean": 0.07453478068527114, "replicates": 8, "sample_sd": 0.005286366286929321, "standard_error": 0.0018690127246618366 }, "log_loss": { "bootstrap_interval": [ 0.23641389014948502, 0.2569978671434998 ], "confidence_level": 0.95, "mean": 0.24691929531401444, "replicates": 8, "sample_sd": 0.01581678268174834, "standard_error": 0.005592077145409099 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11072664539947824, 0.1166726121313378 ], "confidence_level": 0.95, "mean": 0.1136443487378146, "replicates": 8, "sample_sd": 0.004602628198623606, "standard_error": 0.0016272748052635878 }, "log_loss": { "bootstrap_interval": [ 0.3510095654811624, 0.36760874000959315 ], "confidence_level": 0.95, "mean": 0.3590003481495725, "replicates": 8, "sample_sd": 0.0128353060091697, "standard_error": 0.004537965958844168 } }, "learned": { "brier": { "bootstrap_interval": [ 0.07162112422692184, 0.07821051254543092 ], "confidence_level": 0.95, "mean": 0.07491654240118688, "replicates": 8, "sample_sd": 0.005127614173997153, "standard_error": 0.001812885376870822 }, "log_loss": { "bootstrap_interval": [ 0.23832455313728884, 0.25875999357665647 ], "confidence_level": 0.95, "mean": 0.24855943393293967, "replicates": 8, "sample_sd": 0.015755762635393047, "standard_error": 0.005570503301126026 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.07078149796187079, 0.07687153355398134 ], "confidence_level": 0.95, "mean": 0.07398501754104157, "replicates": 8, "sample_sd": 0.0047458923751813676, "standard_error": 0.0016779263406361377 }, "log_loss": { "bootstrap_interval": [ 0.23333722571370843, 0.251324273090046 ], "confidence_level": 0.95, "mean": 0.2425750612306378, "replicates": 8, "sample_sd": 0.013740756263805632, "standard_error": 0.004858090966384245 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.8 ], "oracle_method": "oracle", "population_optimal_scalar": 0.8, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.5-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0007603262878728463, 0.0024257658236504884 ], "confidence_level": 0.95, "mean": 0.0005038803727327595, "replicates": 8, "sample_sd": 0.0026213819949578183, "standard_error": 0.0009267984923574966 }, "log_loss": { "bootstrap_interval": [ -0.0015633480971990784, 0.007182771915796799 ], "confidence_level": 0.95, "mean": 0.0022835596706240052, "replicates": 8, "sample_sd": 0.006962017855225717, "standard_error": 0.0024614450180859638 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0035397614217317305, 0.0025775085305271422 ], "confidence_level": 0.95, "mean": -0.0010396430875729452, "replicates": 8, "sample_sd": 0.004962553040679365, "standard_error": 0.0017545274535311498 }, "log_loss": { "bootstrap_interval": [ -0.014152655617771376, 0.004216225872859547 ], "confidence_level": 0.95, "mean": -0.006038605575166972, "replicates": 8, "sample_sd": 0.014617807549307485, "standard_error": 0.005168175422097614 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0019039555921733465, 0.006236418165910627 ], "confidence_level": 0.95, "mean": 0.0036215670258833467, "replicates": 8, "sample_sd": 0.003581371137395989, "standard_error": 0.001266205908599241 }, "log_loss": { "bootstrap_interval": [ 0.00701042293599638, 0.01905420821204379 ], "confidence_level": 0.95, "mean": 0.012284005676868825, "replicates": 8, "sample_sd": 0.00964379326877278, "standard_error": 0.0034095958083552066 } } }, "discount": { "bootstrap_interval": [ 0.5585024925407917, 0.8663187911157081 ], "confidence_level": 0.95, "mean": 0.7184004011633855, "replicates": 8, "sample_sd": 0.24061401104747757, "standard_error": 0.08506989943008311 }, "discount_below_exchangeable_range_count": 2, "discount_max": 1.0, "discount_min": 0.373790040415257, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18523882986152354, 0.1927319257734103 ], "confidence_level": 0.95, "mean": 0.18895749503585704, "replicates": 8, "sample_sd": 0.005795122035786289, "standard_error": 0.0020488850446540373 }, "log_loss": { "bootstrap_interval": [ 0.5509805282836346, 0.5675775614084594 ], "confidence_level": 0.95, "mean": 0.5591453060130019, "replicates": 8, "sample_sd": 0.012698677722891273, "standard_error": 0.004489660564979482 } }, "independent": { "brier": { "bootstrap_interval": [ 0.18499948405317934, 0.19586328549296253 ], "confidence_level": 0.95, "mean": 0.19050101849616274, "replicates": 8, "sample_sd": 0.008454716561749393, "standard_error": 0.0029891937069116033 }, "log_loss": { "bootstrap_interval": [ 0.5530054659536566, 0.5818226954619655 ], "confidence_level": 0.95, "mean": 0.5674674712587929, "replicates": 8, "sample_sd": 0.022034606405293006, "standard_error": 0.00779040980497961 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19533702596787672, 0.2035607789004858 ], "confidence_level": 0.95, "mean": 0.19920056907551684, "replicates": 8, "sample_sd": 0.00638457994517526, "standard_error": 0.002257289887130531 }, "log_loss": { "bootstrap_interval": [ 0.5737874662513935, 0.5920811836704162 ], "confidence_level": 0.95, "mean": 0.5821224059622203, "replicates": 8, "sample_sd": 0.01413800323726898, "standard_error": 0.0049985389807551284 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1865710984863304, 0.19274629967119367 ], "confidence_level": 0.95, "mean": 0.18946137540858982, "replicates": 8, "sample_sd": 0.004850665579842077, "standard_error": 0.0017149692623872545 }, "log_loss": { "bootstrap_interval": [ 0.5545642965175869, 0.5689402722588809 ], "confidence_level": 0.95, "mean": 0.5614288656836259, "replicates": 8, "sample_sd": 0.011226347408165908, "standard_error": 0.003969113190135068 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18115322513564858, 0.1904001090452381 ], "confidence_level": 0.95, "mean": 0.18583980838270647, "replicates": 8, "sample_sd": 0.007075249687409807, "standard_error": 0.0025014785162777373 }, "log_loss": { "bootstrap_interval": [ 0.5385320971362078, 0.5601524544382576 ], "confidence_level": 0.95, "mean": 0.5491448600067572, "replicates": 8, "sample_sd": 0.01659849065592243, "standard_error": 0.0058684526501321475 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.6666666666666666 ], "oracle_method": "oracle", "population_optimal_scalar": 0.6666666666666666, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.5-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.006396155098841571, -0.003134726989709088 ], "confidence_level": 0.95, "mean": -0.00487223786490789, "replicates": 8, "sample_sd": 0.0025239837375979437, "standard_error": 0.0008923630082300367 }, "log_loss": { "bootstrap_interval": [ -0.021496000055334523, -0.009515464678819363 ], "confidence_level": 0.95, "mean": -0.01581653142076657, "replicates": 8, "sample_sd": 0.009294446401025148, "standard_error": 0.003286083038769892 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0006420336469751854, 0.0004422329772311214 ], "confidence_level": 0.95, "mean": -8.764970243624326e-05, "replicates": 8, "sample_sd": 0.0008440464001549, "standard_error": 0.000298415466592812 }, "log_loss": { "bootstrap_interval": [ -0.00977360166930033, 0.00022421952534871364 ], "confidence_level": 0.95, "mean": -0.004469215719495309, "replicates": 8, "sample_sd": 0.007668213446120943, "standard_error": 0.0027111228636689916 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0009155559976280064, 0.0028969031811291587 ], "confidence_level": 0.95, "mean": 0.0019011226259341464, "replicates": 8, "sample_sd": 0.0015495145079274363, "standard_error": 0.0005478361080512133 }, "log_loss": { "bootstrap_interval": [ 0.007148718969948368, 0.018065374728253324 ], "confidence_level": 0.95, "mean": 0.012552528727441371, "replicates": 8, "sample_sd": 0.008447388547616981, "standard_error": 0.002986602862668774 } } }, "discount": { "bootstrap_interval": [ 0.613391794610286, 0.9129833568627791 ], "confidence_level": 0.95, "mean": 0.765705226160751, "replicates": 8, "sample_sd": 0.23272318262548666, "standard_error": 0.08228007028689846 }, "discount_below_exchangeable_range_count": 1, "discount_max": 1.0, "discount_min": 0.4273100499060817, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0913402498035108, 0.09693949379876836 ], "confidence_level": 0.95, "mean": 0.09428464990561969, "replicates": 8, "sample_sd": 0.004326515223558639, "standard_error": 0.0015296541267425725 }, "log_loss": { "bootstrap_interval": [ 0.3043197508030417, 0.3188858897103818 ], "confidence_level": 0.95, "mean": 0.31180777398613396, "replicates": 8, "sample_sd": 0.011190646519745377, "standard_error": 0.003956491019986796 } }, "independent": { "brier": { "bootstrap_interval": [ 0.08558772448399744, 0.0932980092362027 ], "confidence_level": 0.95, "mean": 0.08950006174314803, "replicates": 8, "sample_sd": 0.0059719390035574815, "standard_error": 0.0021113992831239643 }, "log_loss": { "bootstrap_interval": [ 0.28961511034917153, 0.3105925476053884 ], "confidence_level": 0.95, "mean": 0.30046045828486273, "replicates": 8, "sample_sd": 0.016498216108905145, "standard_error": 0.005833000244043982 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10882338794860694, 0.11584098616955958 ], "confidence_level": 0.95, "mean": 0.11230572728599801, "replicates": 8, "sample_sd": 0.005498152457137541, "standard_error": 0.0019438904432197166 }, "log_loss": { "bootstrap_interval": [ 0.34671601562647825, 0.3654353949632302 ], "confidence_level": 0.95, "mean": 0.3558041579520046, "replicates": 8, "sample_sd": 0.014597153738659536, "standard_error": 0.005160873197314361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.08589058419009528, 0.09294232779360134 ], "confidence_level": 0.95, "mean": 0.08941241204071179, "replicates": 8, "sample_sd": 0.0055144873464364145, "standard_error": 0.0019496656987162992 }, "log_loss": { "bootstrap_interval": [ 0.28466469608612055, 0.3064063656327777 ], "confidence_level": 0.95, "mean": 0.2959912425653674, "replicates": 8, "sample_sd": 0.016542341478764797, "standard_error": 0.005848600918169044 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08425763065667688, 0.09056964038036744 ], "confidence_level": 0.95, "mean": 0.08751128941477765, "replicates": 8, "sample_sd": 0.004995192645886761, "standard_error": 0.0017660672966198503 }, "log_loss": { "bootstrap_interval": [ 0.27479121754042424, 0.2921569457135886 ], "confidence_level": 0.95, "mean": 0.283438713837926, "replicates": 8, "sample_sd": 0.013560114233593532, "standard_error": 0.004794224364119104 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.6666666666666666 ], "oracle_method": "oracle", "population_optimal_scalar": 0.6666666666666666, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.75-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0012952304183322639, 0.0054400098178643275 ], "confidence_level": 0.95, "mean": 0.00306292786065665, "replicates": 8, "sample_sd": 0.0032989087676273576, "standard_error": 0.0011663403800525305 }, "log_loss": { "bootstrap_interval": [ 0.0040755845721663075, 0.017248832683527747 ], "confidence_level": 0.95, "mean": 0.010170843217484016, "replicates": 8, "sample_sd": 0.010333126918650837, "standard_error": 0.0036533120575196307 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.008171612673713344, 0.0004123371293520174 ], "confidence_level": 0.95, "mean": -0.004615177399799393, "replicates": 8, "sample_sd": 0.006732395993585687, "standard_error": 0.0023802614303487916 }, "log_loss": { "bootstrap_interval": [ -0.03387608826390854, -0.008295845140576458 ], "confidence_level": 0.95, "mean": -0.022612744822879366, "replicates": 8, "sample_sd": 0.019906786483934978, "standard_error": 0.007038111857211565 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0018011491538508333, 0.006750197781107438 ], "confidence_level": 0.95, "mean": 0.0038492493567608144, "replicates": 8, "sample_sd": 0.003928724294057751, "standard_error": 0.0013890137948702837 }, "log_loss": { "bootstrap_interval": [ 0.006100056040646483, 0.02110019345176517 ], "confidence_level": 0.95, "mean": 0.012853807802033296, "replicates": 8, "sample_sd": 0.01170190209595962, "standard_error": 0.00413724716241706 } } }, "discount": { "bootstrap_interval": [ 0.48262217308816124, 0.7753317124609501 ], "confidence_level": 0.95, "mean": 0.6277058156420903, "replicates": 8, "sample_sd": 0.230527951100033, "standard_error": 0.08150393873793707 }, "discount_below_exchangeable_range_count": 3, "discount_max": 1.0, "discount_min": 0.3207534976616162, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18919486010179734, 0.1978980689115086 ], "confidence_level": 0.95, "mean": 0.193631444106457, "replicates": 8, "sample_sd": 0.006665396374755828, "standard_error": 0.002356573487943038 }, "log_loss": { "bootstrap_interval": [ 0.5596160179461703, 0.5782767343062278 ], "confidence_level": 0.95, "mean": 0.5689248776631053, "replicates": 8, "sample_sd": 0.014517332085666159, "standard_error": 0.005132651981255793 } }, "independent": { "brier": { "bootstrap_interval": [ 0.19449690073658454, 0.20756218517228928 ], "confidence_level": 0.95, "mean": 0.20130954936691303, "replicates": 8, "sample_sd": 0.010084227767484785, "standard_error": 0.003565312918709085 }, "log_loss": { "bootstrap_interval": [ 0.5846371452071616, 0.6176308543663435 ], "confidence_level": 0.95, "mean": 0.6017084657034687, "replicates": 8, "sample_sd": 0.025237619592710637, "standard_error": 0.008922845977506082 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1941935526773648, 0.20339267184464438 ], "confidence_level": 0.95, "mean": 0.19869529628463112, "replicates": 8, "sample_sd": 0.007071188646351022, "standard_error": 0.0025000427214420654 }, "log_loss": { "bootstrap_interval": [ 0.5711933601876992, 0.5912517894068656 ], "confidence_level": 0.95, "mean": 0.58080379894007, "replicates": 8, "sample_sd": 0.015418508102089555, "standard_error": 0.005451265817383624 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1932066803597837, 0.2005032600852255 ], "confidence_level": 0.95, "mean": 0.19669437196711365, "replicates": 8, "sample_sd": 0.005548786008539367, "standard_error": 0.001961792106995611 }, "log_loss": { "bootstrap_interval": [ 0.5697544005042678, 0.5885924805851682 ], "confidence_level": 0.95, "mean": 0.5790957208805894, "replicates": 8, "sample_sd": 0.014424634744998922, "standard_error": 0.0050998785221639115 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18801709094941496, 0.19752377247145064 ], "confidence_level": 0.95, "mean": 0.1928451226103528, "replicates": 8, "sample_sd": 0.007390822682987622, "standard_error": 0.00261305041884395 }, "log_loss": { "bootstrap_interval": [ 0.5554246816128672, 0.5769501751731048 ], "confidence_level": 0.95, "mean": 0.5662419130785561, "replicates": 8, "sample_sd": 0.016526618901939624, "standard_error": 0.00584304214782364 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5714285714285714 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5714285714285714, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho0.75-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0006965478038484241, 0.0009574005571693685 ], "confidence_level": 0.95, "mean": 1.8241298248254656e-05, "replicates": 8, "sample_sd": 0.001306918663892909, "standard_error": 0.0004620655248489691 }, "log_loss": { "bootstrap_interval": [ -4.5549993954924206e-05, 0.010521387221488274 ], "confidence_level": 0.95, "mean": 0.004817911099935325, "replicates": 8, "sample_sd": 0.008378787758758902, "standard_error": 0.002962348821170627 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0033230825884239343, -0.001202155195167997 ], "confidence_level": 0.95, "mean": -0.0023429576046073864, "replicates": 8, "sample_sd": 0.0017140470451852022, "standard_error": 0.0006060071444616105 }, "log_loss": { "bootstrap_interval": [ -0.030541331878247056, -0.01602128722392166 ], "confidence_level": 0.95, "mean": -0.023941504403188325, "replicates": 8, "sample_sd": 0.011438736739889908, "standard_error": 0.004044204158491927 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0013566181162367587, 0.0027416306934589588 ], "confidence_level": 0.95, "mean": 0.0020318911706699114, "replicates": 8, "sample_sd": 0.0010628636479980487, "standard_error": 0.0003757790464880459 }, "log_loss": { "bootstrap_interval": [ 0.008710521224387329, 0.019843067552042973 ], "confidence_level": 0.95, "mean": 0.013715417462455193, "replicates": 8, "sample_sd": 0.008813988804542811, "standard_error": 0.0031162156264972663 } } }, "discount": { "bootstrap_interval": [ 0.5281502048289026, 0.8118138311757468 ], "confidence_level": 0.95, "mean": 0.6704726063612535, "replicates": 8, "sample_sd": 0.21749262037784453, "standard_error": 0.07689525336360267 }, "discount_below_exchangeable_range_count": 3, "discount_max": 1.0, "discount_min": 0.37052853376047323, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09774933851700454, 0.1049894261874316 ], "confidence_level": 0.95, "mean": 0.10164997431873746, "replicates": 8, "sample_sd": 0.005588103409923692, "standard_error": 0.001975692907564356 }, "log_loss": { "bootstrap_interval": [ 0.31836228242012876, 0.33586746279193175 ], "confidence_level": 0.95, "mean": 0.32768807132685196, "replicates": 8, "sample_sd": 0.013437872955350302, "standard_error": 0.004751005545725755 } }, "independent": { "brier": { "bootstrap_interval": [ 0.09879803824971697, 0.1084075124837579 ], "confidence_level": 0.95, "mean": 0.1040111732215931, "replicates": 8, "sample_sd": 0.007478995509768442, "standard_error": 0.0026442242207105025 }, "log_loss": { "bootstrap_interval": [ 0.3427242220618254, 0.3687062375377818 ], "confidence_level": 0.95, "mean": 0.3564474868299756, "replicates": 8, "sample_sd": 0.01985861735391105, "standard_error": 0.007021081497969677 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10697188025842762, 0.1152341975524816 ], "confidence_level": 0.95, "mean": 0.11121122109872107, "replicates": 8, "sample_sd": 0.006408123641965544, "standard_error": 0.002265613840957836 }, "log_loss": { "bootstrap_interval": [ 0.34255974597075545, 0.3628515029980437 ], "confidence_level": 0.95, "mean": 0.3526925037658831, "replicates": 8, "sample_sd": 0.01593143496684411, "standard_error": 0.005632612849543975 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09751437591360068, 0.10527468269105807 ], "confidence_level": 0.95, "mean": 0.10166821561698572, "replicates": 8, "sample_sd": 0.006085087066467598, "standard_error": 0.002151403164404897 }, "log_loss": { "bootstrap_interval": [ 0.3210002533537087, 0.3425261104616305 ], "confidence_level": 0.95, "mean": 0.3325059824267873, "replicates": 8, "sample_sd": 0.016794834212095027, "standard_error": 0.00593787058013811 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.09534003346542441, 0.10329108342149576 ], "confidence_level": 0.95, "mean": 0.0996363244463158, "replicates": 8, "sample_sd": 0.006140297003867389, "standard_error": 0.0021709228249670357 }, "log_loss": { "bootstrap_interval": [ 0.30874190159936277, 0.3282702828543991 ], "confidence_level": 0.95, "mean": 0.3187905649643321, "replicates": 8, "sample_sd": 0.015119918477638945, "standard_error": 0.005345698443263139 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5714285714285714 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5714285714285714, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho1-a0.5": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0010466126773966584, 0.0051007184623462105 ], "confidence_level": 0.95, "mean": 0.002669671448623418, "replicates": 8, "sample_sd": 0.0032450327229714983, "standard_error": 0.0011472923217926967 }, "log_loss": { "bootstrap_interval": [ 0.003493109673419592, 0.014305943855460086 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.012836182553728665, -0.004392582293782529 ], "confidence_level": 0.95, "mean": -0.009340332526677219, "replicates": 8, "sample_sd": 0.0067121968965600425, "standard_error": 0.0023731199711084527 }, "log_loss": { "bootstrap_interval": [ -0.05832759769828064, -0.03231211957914419 ], "confidence_level": 0.95, "mean": -0.047203650372559566, "replicates": 8, "sample_sd": 0.02073933820417222, "standard_error": 0.007332463340745705 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.001046612677396655, 0.005100718462346207 ], "confidence_level": 0.95, "mean": 0.0026696714486234147, "replicates": 8, "sample_sd": 0.0032450327229714983, "standard_error": 0.0011472923217926967 }, "log_loss": { "bootstrap_interval": [ 0.003493109673419592, 0.014305943855460086 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } } }, "discount": { "bootstrap_interval": [ 0.43617399746176116, 0.6389763532869773 ], "confidence_level": 0.95, "mean": 0.5384604292867611, "replicates": 8, "sample_sd": 0.15767053463761949, "standard_error": 0.05574495211778458 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.7592377547385127, "discount_min": 0.30747155079197697, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1930869216585636, 0.2026219615826795 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.0074192493067776494, "standard_error": 0.002623100748068034 }, "log_loss": { "bootstrap_interval": [ 0.5684099892450882, 0.5891359582670682 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2026408735335485, 0.21687400065019585 ], "confidence_level": 0.95, "mean": 0.21009356776013377, "replicates": 8, "sample_sd": 0.011066272709834565, "standard_error": 0.0039125182377918255 }, "log_loss": { "bootstrap_interval": [ 0.6147360554874959, 0.652332005702125 ], "confidence_level": 0.95, "mean": 0.6346886963756371, "replicates": 8, "sample_sd": 0.02923322728121046, "standard_error": 0.010335506623255749 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1930308678167527, 0.20263615560894585 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5685972002799563, 0.5893518842277922 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1965772814757764, 0.20481653720063359 ], "confidence_level": 0.95, "mean": 0.20075323523345656, "replicates": 8, "sample_sd": 0.006395098492831276, "standard_error": 0.0022610087553184325 }, "log_loss": { "bootstrap_interval": [ 0.5778569391989096, 0.5972881488218128 ], "confidence_level": 0.95, "mean": 0.5874850460030775, "replicates": 8, "sample_sd": 0.015094497522713133, "standard_error": 0.005336710778456999 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19312057345363048, 0.20261807968625617 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5684389236342737, 0.5892435495742444 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n2-rho1-a1": { "condition": { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0007918465849649021, 0.0034272831658810165 ], "confidence_level": 0.95, "mean": 0.002033601459410842, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.004735454767358289, 0.020092521249392342 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922936, "standard_error": 0.004312314006328674 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.006935366216277524, -0.0025246383317630524 ], "confidence_level": 0.95, "mean": -0.00489358409724047, "replicates": 8, "sample_sd": 0.003451298153359754, "standard_error": 0.0012202181640686455 }, "log_loss": { "bootstrap_interval": [ -0.06803280216879348, -0.039180261710854036 ], "confidence_level": 0.95, "mean": -0.05465804199398918, "replicates": 8, "sample_sd": 0.02192521159322118, "standard_error": 0.007751732898258302 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0007918465849649021, 0.0034272831658810165 ], "confidence_level": 0.95, "mean": 0.002033601459410842, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.004735454767358289, 0.020092521249392342 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922936, "standard_error": 0.004312314006328674 } } }, "discount": { "bootstrap_interval": [ 0.45369654990566677, 0.6897335252267323 ], "confidence_level": 0.95, "mean": 0.5649055321917836, "replicates": 8, "sample_sd": 0.18320612302033906, "standard_error": 0.06477314597128929 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.906070528922815, "discount_min": 0.3525587093027146, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 2 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1048348370492453, 0.11424138826303429 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.3367011473205406, 0.36006994274458043 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "independent": { "brier": { "bootstrap_interval": [ 0.11003141118847147, 0.12247899343905197 ], "confidence_level": 0.95, "mean": 0.11683882797948877, "replicates": 8, "sample_sd": 0.009694725914515976, "standard_error": 0.0034276032179495998 }, "log_loss": { "bootstrap_interval": [ 0.39495271154940226, 0.433057497254136 ], "confidence_level": 0.95, "mean": 0.41552912117140683, "replicates": 8, "sample_sd": 0.029979503480814934, "standard_error": 0.010599355103944972 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1047584263609172, 0.11428741949844008 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.3373435665567529, 0.36006994274458043 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10729961082206102, 0.11603068862123402 ], "confidence_level": 0.95, "mean": 0.1119452438822483, "replicates": 8, "sample_sd": 0.006750192655061164, "standard_error": 0.002386553500354687 }, "log_loss": { "bootstrap_interval": [ 0.3485902539406704, 0.37182548800928295 ], "confidence_level": 0.95, "mean": 0.3608710791774176, "replicates": 8, "sample_sd": 0.018016476853949994, "standard_error": 0.006369786478259258 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10478867340606127, 0.11424900267635443 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.33681558465182565, 0.35990004876905346 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.15107210954239358, -0.1486733875268117 ], "confidence_level": 0.95, "mean": -0.14985845594213087, "replicates": 8, "sample_sd": 0.0018864418341948705, "standard_error": 0.0006669579066365908 }, "log_loss": { "bootstrap_interval": [ -0.4769181285266298, -0.4663357112594746 ], "confidence_level": 0.95, "mean": -0.47187283836025223, "replicates": 8, "sample_sd": 0.008140220222125636, "standard_error": 0.0028780024597084503 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0008023995678036523 ], "confidence_level": 0.95, "mean": 0.0003205310968199646, "replicates": 8, "sample_sd": 0.0006864730108472765, "standard_error": 0.0002427048605358278 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.00769504033759505 ], "confidence_level": 0.95, "mean": 0.003048153727093412, "replicates": 8, "sample_sd": 0.0065984856065133615, "standard_error": 0.002332916958963713 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0008023995678036523 ], "confidence_level": 0.95, "mean": 0.0003205310968199646, "replicates": 8, "sample_sd": 0.0006864730108472765, "standard_error": 0.0002427048605358278 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.00769504033759505 ], "confidence_level": 0.95, "mean": 0.003048153727093412, "replicates": 8, "sample_sd": 0.0065984856065133615, "standard_error": 0.002332916958963713 } } }, "discount": { "bootstrap_interval": [ 0.7453785780722573, 1.0 ], "confidence_level": 0.95, "mean": 0.892168185573547, "replicates": 8, "sample_sd": 0.20396305057188246, "standard_error": 0.07211182808543641 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.49075715614451443, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1592781134387352, 0.16026970538712704 ], "confidence_level": 0.95, "mean": 0.15970361261020466, "replicates": 8, "sample_sd": 0.0007859887127870395, "standard_error": 0.00027788897437390064 }, "log_loss": { "bootstrap_interval": [ 0.5079355480984012, 0.5099292110487581 ], "confidence_level": 0.95, "mean": 0.5087957709379497, "replicates": 8, "sample_sd": 0.0015924159990206646, "standard_error": 0.0005630040756887312 } }, "independent": { "brier": { "bootstrap_interval": [ 0.008614030132493199, 0.0105121831413457 ], "confidence_level": 0.95, "mean": 0.00952462557125382, "replicates": 8, "sample_sd": 0.001449455407654353, "standard_error": 0.0005124598738899523 }, "log_loss": { "bootstrap_interval": [ 0.030748875048191945, 0.03710373597087798 ], "confidence_level": 0.95, "mean": 0.03387477885060407, "replicates": 8, "sample_sd": 0.004904676677562597, "standard_error": 0.001734065069116009 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19760412168107191, 0.2033033126543609 ], "confidence_level": 0.95, "mean": 0.200401419084981, "replicates": 8, "sample_sd": 0.004402965745277611, "standard_error": 0.00155668346790894 }, "log_loss": { "bootstrap_interval": [ 0.5785676935408931, 0.5918256195999996 ], "confidence_level": 0.95, "mean": 0.5850450001139459, "replicates": 8, "sample_sd": 0.01030599781392939, "standard_error": 0.0036437204705616027 } }, "learned": { "brier": { "bootstrap_interval": [ 0.008756701604374596, 0.0109974121775078 ], "confidence_level": 0.95, "mean": 0.009845156668073785, "replicates": 8, "sample_sd": 0.0017135921978501407, "standard_error": 0.0006058463316440972 }, "log_loss": { "bootstrap_interval": [ 0.03211109017625576, 0.04272505389487133 ], "confidence_level": 0.95, "mean": 0.03692293257769748, "replicates": 8, "sample_sd": 0.008281236458015984, "standard_error": 0.002927859228036184 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.00862949178817883, 0.010499792761652848 ], "confidence_level": 0.95, "mean": 0.00952462557125382, "replicates": 8, "sample_sd": 0.001449455407654353, "standard_error": 0.0005124598738899523 }, "log_loss": { "bootstrap_interval": [ 0.030650572306834527, 0.037169144090925685 ], "confidence_level": 0.95, "mean": 0.03387477885060407, "replicates": 8, "sample_sd": 0.004904676677562597, "standard_error": 0.001734065069116009 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05356605493889817, -0.05275035744625254 ], "confidence_level": 0.95, "mean": -0.05309609893560684, "replicates": 8, "sample_sd": 0.0006411138221028803, "standard_error": 0.00022666796556068623 }, "log_loss": { "bootstrap_interval": [ -0.2568830854541032, -0.2548236940114577 ], "confidence_level": 0.95, "mean": -0.2556987722717943, "replicates": 8, "sample_sd": 0.0016356620434570597, "standard_error": 0.0005782938613289662 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0 ], "confidence_level": 0.95, "mean": 0.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "discount": { "bootstrap_interval": [ 1.0, 1.0 ], "confidence_level": 0.95, "mean": 1.0, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 1.0, "discount_one_count": 8, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.05274996101064155, 0.053556621894278345 ], "confidence_level": 0.95, "mean": 0.05309609895076026, "replicates": 8, "sample_sd": 0.000641113813636839, "standard_error": 0.00022666796256748866 }, "log_loss": { "bootstrap_interval": [ 0.25483249564663735, 0.256904492821961 ], "confidence_level": 0.95, "mean": 0.2556988439970225, "replicates": 8, "sample_sd": 0.001635636136859785, "standard_error": 0.0005782847019636609 } }, "independent": { "brier": { "bootstrap_interval": [ 6.81875198254286e-13, 4.113654375912214e-11 ], "confidence_level": 0.95, "mean": 1.515341119382969e-11, "replicates": 8, "sample_sd": 3.6117451720655275e-11, "standard_error": 1.2769447515426541e-11 }, "log_loss": { "bootstrap_interval": [ 1.3537275474188955e-08, 1.5714305204312363e-07 ], "confidence_level": 0.95, "mean": 7.17252282043819e-08, "replicates": 8, "sample_sd": 1.1540133969737512e-07, "standard_error": 4.080053492901313e-08 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11215088857157363, 0.11756306364863406 ], "confidence_level": 0.95, "mean": 0.11474074946836885, "replicates": 8, "sample_sd": 0.004152437295533535, "standard_error": 0.0014681082850618453 }, "log_loss": { "bootstrap_interval": [ 0.3550733056390166, 0.3698543599790407 ], "confidence_level": 0.95, "mean": 0.36231777974411405, "replicates": 8, "sample_sd": 0.011648816374946308, "standard_error": 0.004118478525760715 } }, "learned": { "brier": { "bootstrap_interval": [ 6.823880946974933e-13, 4.11382989798128e-11 ], "confidence_level": 0.95, "mean": 1.515341119382969e-11, "replicates": 8, "sample_sd": 3.6117451720655275e-11, "standard_error": 1.2769447515426541e-11 }, "log_loss": { "bootstrap_interval": [ 1.3234188365690254e-08, 1.5584855564646516e-07 ], "confidence_level": 0.95, "mean": 7.17252282043819e-08, "replicates": 8, "sample_sd": 1.1540133969737512e-07, "standard_error": 4.080053492901313e-08 } }, "oracle": { "brier": { "bootstrap_interval": [ 6.802527838177465e-13, 4.11382989798128e-11 ], "confidence_level": 0.95, "mean": 1.515341119382969e-11, "replicates": 8, "sample_sd": 3.6117451720655275e-11, "standard_error": 1.2769447515426541e-11 }, "log_loss": { "bootstrap_interval": [ 1.3271391261995623e-08, 1.586856118455444e-07 ], "confidence_level": 0.95, "mean": 7.17252282043819e-08, "replicates": 8, "sample_sd": 1.1540133969737512e-07, "standard_error": 4.080053492901313e-08 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.1-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.09755734152511904, -0.09263032644919973 ], "confidence_level": 0.95, "mean": -0.09519641335045248, "replicates": 8, "sample_sd": 0.0037988877637085824, "standard_error": 0.0013431096493424686 }, "log_loss": { "bootstrap_interval": [ -0.2942377239661178, -0.24750041770045653 ], "confidence_level": 0.95, "mean": -0.27379276957353355, "replicates": 8, "sample_sd": 0.03728898298483181, "standard_error": 0.013183646366062179 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.008825082107775967, -0.0027169870061450286 ], "confidence_level": 0.95, "mean": -0.006047771937160954, "replicates": 8, "sample_sd": 0.00481730359559923, "standard_error": 0.0017031740197412765 }, "log_loss": { "bootstrap_interval": [ -0.12422824049510284, -0.07176508351834543 ], "confidence_level": 0.95, "mean": -0.10335068511392552, "replicates": 8, "sample_sd": 0.04332366212921287, "standard_error": 0.01531722763870062 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.00040785410763557984, 0.004793785638716736 ], "confidence_level": 0.95, "mean": 0.002416339053757455, "replicates": 8, "sample_sd": 0.0033787821369942028, "standard_error": 0.0011945798806102875 }, "log_loss": { "bootstrap_interval": [ 0.0031574973258619507, 0.04953579416502603 ], "confidence_level": 0.95, "mean": 0.02217424912072424, "replicates": 8, "sample_sd": 0.038444591230219316, "standard_error": 0.013592215579416476 } } }, "discount": { "bootstrap_interval": [ 0.33748163413610505, 0.6389289699790087 ], "confidence_level": 0.95, "mean": 0.4683659064080656, "replicates": 8, "sample_sd": 0.23825303223452662, "standard_error": 0.08423516736564543 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.1895430446609659, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16198624658949282, 0.1647662254023048 ], "confidence_level": 0.95, "mean": 0.1634178240838376, "replicates": 8, "sample_sd": 0.0021365737666803877, "standard_error": 0.0007553928994624932 }, "log_loss": { "bootstrap_interval": [ 0.5110932422602714, 0.5170092838424951 ], "confidence_level": 0.95, "mean": 0.5141426673704919, "replicates": 8, "sample_sd": 0.004583668934305514, "standard_error": 0.0016205716930807721 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07140101209671856, 0.07679950914959399 ], "confidence_level": 0.95, "mean": 0.07426918267054608, "replicates": 8, "sample_sd": 0.0041516837248578254, "standard_error": 0.0014678418575943964 }, "log_loss": { "bootstrap_interval": [ 0.32552849470026596, 0.36093510548872265 ], "confidence_level": 0.95, "mean": 0.3437005829108839, "replicates": 8, "sample_sd": 0.027443562538408517, "standard_error": 0.00970276458541288 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19723430109388385, 0.203412577697134 ], "confidence_level": 0.95, "mean": 0.20005045972324384, "replicates": 8, "sample_sd": 0.004854666888269692, "standard_error": 0.0017163839385486471 }, "log_loss": { "bootstrap_interval": [ 0.577817425322906, 0.5919684839338384 ], "confidence_level": 0.95, "mean": 0.5842897509669385, "replicates": 8, "sample_sd": 0.01106838076050973, "standard_error": 0.003913263546255573 } }, "learned": { "brier": { "bootstrap_interval": [ 0.06507095004200343, 0.07150954514923664 ], "confidence_level": 0.95, "mean": 0.06822141073338513, "replicates": 8, "sample_sd": 0.004759398624089481, "standard_error": 0.001682701520731798 }, "log_loss": { "bootstrap_interval": [ 0.2186728841193389, 0.26792166710330506 ], "confidence_level": 0.95, "mean": 0.24034989779695837, "replicates": 8, "sample_sd": 0.038557019149599775, "standard_error": 0.013631964851510784 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06358421429422943, 0.06773232925327653 ], "confidence_level": 0.95, "mean": 0.06580507167962768, "replicates": 8, "sample_sd": 0.0032616571238132156, "standard_error": 0.0011531699350768676 }, "log_loss": { "bootstrap_interval": [ 0.21065651537147906, 0.22502614649910205 ], "confidence_level": 0.95, "mean": 0.21817564867623412, "replicates": 8, "sample_sd": 0.011068078698476315, "standard_error": 0.00391315675119949 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3448275862068965 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3448275862068966, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.1-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.055505538710187645, -0.05382656084960272 ], "confidence_level": 0.95, "mean": -0.05460177248989044, "replicates": 8, "sample_sd": 0.001303872621318334, "standard_error": 0.0004609885861688366 }, "log_loss": { "bootstrap_interval": [ -0.24175407540965982, -0.2364529817835314 ], "confidence_level": 0.95, "mean": -0.23890317120774313, "replicates": 8, "sample_sd": 0.0041392429210899105, "standard_error": 0.0014634433692405444 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0005673509907820742 ], "confidence_level": 0.95, "mean": 0.00018911699692735807, "replicates": 8, "sample_sd": 0.0005349036438598814, "standard_error": 0.00018911699692735805 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0017991756288226026 ], "confidence_level": 0.95, "mean": 0.0005997252096075342, "replicates": 8, "sample_sd": 0.0016962790502480442, "standard_error": 0.0005997252096075342 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 7.845802125629295e-07, 0.0008457303425593647 ], "confidence_level": 0.95, "mean": 0.0003785175262773818, "replicates": 8, "sample_sd": 0.0006673486909166191, "standard_error": 0.00023594339238155334 }, "log_loss": { "bootstrap_interval": [ 0.008884738889802422, 0.01182586385521961 ], "confidence_level": 0.95, "mean": 0.010195478649231808, "replicates": 8, "sample_sd": 0.0022623631934502474, "standard_error": 0.0007998661777977614 } } }, "discount": { "bootstrap_interval": [ 0.677119412015983, 1.0 ], "confidence_level": 0.95, "mean": 0.892373137338661, "replicates": 8, "sample_sd": 0.30441473770266414, "standard_error": 0.10762686266133899 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.138985098709288, "discount_one_count": 7, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.056943773827393955, 0.058626936116831 ], "confidence_level": 0.95, "mean": 0.05784408389195513, "replicates": 8, "sample_sd": 0.0013285092428676052, "standard_error": 0.0004696989472503448 }, "log_loss": { "bootstrap_interval": [ 0.25730462199591375, 0.2623364079083419 ], "confidence_level": 0.95, "mean": 0.25998965944564817, "replicates": 8, "sample_sd": 0.0038653252481761348, "standard_error": 0.0013665988472384596 } }, "independent": { "brier": { "bootstrap_interval": [ 0.002450108945769492, 0.0036543732931881793 ], "confidence_level": 0.95, "mean": 0.003053194405137328, "replicates": 8, "sample_sd": 0.0009272929139546851, "standard_error": 0.00032784755380179574 }, "log_loss": { "bootstrap_interval": [ 0.018577200747302684, 0.022397903769327036 ], "confidence_level": 0.95, "mean": 0.020486763028297494, "replicates": 8, "sample_sd": 0.0029387655240846683, "standard_error": 0.0010390105151987536 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11173955451659755, 0.11722222888026873 ], "confidence_level": 0.95, "mean": 0.11440040546560537, "replicates": 8, "sample_sd": 0.004214884081710997, "standard_error": 0.0014901865580465399 }, "log_loss": { "bootstrap_interval": [ 0.3536159962058184, 0.36854026863347833 ], "confidence_level": 0.95, "mean": 0.3609438993647717, "replicates": 8, "sample_sd": 0.011549219867090432, "standard_error": 0.00408326584271702 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0024787683487703226, 0.004095515033909371 ], "confidence_level": 0.95, "mean": 0.0032423114020646866, "replicates": 8, "sample_sd": 0.0012694213397474437, "standard_error": 0.0004488082187591648 }, "log_loss": { "bootstrap_interval": [ 0.018692450003270755, 0.024039576606458543 ], "confidence_level": 0.95, "mean": 0.02108648823790503, "replicates": 8, "sample_sd": 0.0041523850005372835, "standard_error": 0.0014680897959886093 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.0024380490157489055, 0.003300642844633621 ], "confidence_level": 0.95, "mean": 0.0028637938757873046, "replicates": 8, "sample_sd": 0.0006702144556169371, "standard_error": 0.0002369565932079933 }, "log_loss": { "bootstrap_interval": [ 0.009709967459063383, 0.012218092413322454 ], "confidence_level": 0.95, "mean": 0.010891009588673221, "replicates": 8, "sample_sd": 0.0019752588155006368, "standard_error": 0.0006983594515195038 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3448275862068965 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3448275862068966, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.25-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04847567677324291, -0.04067598955904464 ], "confidence_level": 0.95, "mean": -0.044733487999480626, "replicates": 8, "sample_sd": 0.00606482464024975, "standard_error": 0.0021442393149139305 }, "log_loss": { "bootstrap_interval": [ -0.14211356051202706, -0.002834924679529511 ], "confidence_level": 0.95, "mean": -0.092245567991522, "replicates": 8, "sample_sd": 0.1235502906155063, "standard_error": 0.04368162415589658 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03313929911529556, -0.02001707196698579 ], "confidence_level": 0.95, "mean": -0.027651718464373864, "replicates": 8, "sample_sd": 0.010344370997431504, "standard_error": 0.0036572874396966332 }, "log_loss": { "bootstrap_interval": [ -0.641519319975274, -0.45009801103266367 ], "confidence_level": 0.95, "mean": -0.563717636853612, "replicates": 8, "sample_sd": 0.15745353041565718, "standard_error": 0.05566822953933675 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0005733527976939223, 0.008020723210540802 ], "confidence_level": 0.95, "mean": 0.003489398528473339, "replicates": 8, "sample_sd": 0.00628762090423549, "standard_error": 0.002223009689457603 }, "log_loss": { "bootstrap_interval": [ 0.0033159137294285585, 0.1418887825116169 ], "confidence_level": 0.95, "mean": 0.051462720719922764, "replicates": 8, "sample_sd": 0.12655179574171624, "standard_error": 0.0447428164701512 } } }, "discount": { "bootstrap_interval": [ 0.16204244933959777, 0.40704519749853607 ], "confidence_level": 0.95, "mean": 0.25632910587560154, "replicates": 8, "sample_sd": 0.2087906936108591, "standard_error": 0.07381865765044061 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.7627274556429648, "discount_min": 0.10808545985248766, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16677849695987448, 0.17105287250248746 ], "confidence_level": 0.95, "mean": 0.16900915824335286, "replicates": 8, "sample_sd": 0.003299724941822634, "standard_error": 0.0011666289412065852 }, "log_loss": { "bootstrap_interval": [ 0.5180244777028375, 0.5272638965611797 ], "confidence_level": 0.95, "mean": 0.5227351317810492, "replicates": 8, "sample_sd": 0.007097279291544815, "standard_error": 0.002509267157513097 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14461743506450195, 0.15877044983222222 ], "confidence_level": 0.95, "mean": 0.1519273887082461, "replicates": 8, "sample_sd": 0.011062561284585687, "standard_error": 0.003911206050811152 }, "log_loss": { "bootstrap_interval": [ 0.9423057401064342, 1.03978852915847 ], "confidence_level": 0.95, "mean": 0.9942072006431393, "replicates": 8, "sample_sd": 0.0758522252335411, "standard_error": 0.02681781141536313 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19657295490487192, 0.2036289032871928 ], "confidence_level": 0.95, "mean": 0.19971191477641081, "replicates": 8, "sample_sd": 0.005506869817303598, "standard_error": 0.001946972495463449 }, "log_loss": { "bootstrap_interval": [ 0.576322538004945, 0.5922519916553793 ], "confidence_level": 0.95, "mean": 0.5834449207252018, "replicates": 8, "sample_sd": 0.012400593424502377, "standard_error": 0.004384271850601471 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11870799518022826, 0.12934159030260564 ], "confidence_level": 0.95, "mean": 0.12427567024387223, "replicates": 8, "sample_sd": 0.0081971195828844, "standard_error": 0.0028981194216273014 }, "log_loss": { "bootstrap_interval": [ 0.3769439700014001, 0.5200866723143593 ], "confidence_level": 0.95, "mean": 0.43048956378952724, "replicates": 8, "sample_sd": 0.12463649832336249, "standard_error": 0.04406565657389769 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11645212540650685, 0.12459833008944872 ], "confidence_level": 0.95, "mean": 0.12078627171539891, "replicates": 8, "sample_sd": 0.006265435565136483, "standard_error": 0.002215165987597688 }, "log_loss": { "bootstrap_interval": [ 0.3677761672056507, 0.3887505320038863 ], "confidence_level": 0.95, "mean": 0.3790268430696045, "replicates": 8, "sample_sd": 0.016198122456617566, "standard_error": 0.005726901115782189 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.17391304347826086 ], "oracle_method": "oracle", "population_optimal_scalar": 0.17391304347826086, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.25-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04497290220799047, -0.040217128272972524 ], "confidence_level": 0.95, "mean": -0.04258746151721853, "replicates": 8, "sample_sd": 0.0037240389689464815, "standard_error": 0.0013166466041725077 }, "log_loss": { "bootstrap_interval": [ -0.19223387620669152, -0.10597752273816513 ], "confidence_level": 0.95, "mean": -0.15293088877454192, "replicates": 8, "sample_sd": 0.06784079825267167, "standard_error": 0.02398534424278631 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.004702218008915961, -0.0011248248839066067 ], "confidence_level": 0.95, "mean": -0.0029702959226386616, "replicates": 8, "sample_sd": 0.0028055142725191442, "standard_error": 0.0009918990834069652 }, "log_loss": { "bootstrap_interval": [ -0.15995059146972554, -0.06628391861541603 ], "confidence_level": 0.95, "mean": -0.11613723659553896, "replicates": 8, "sample_sd": 0.0741607005847966, "standard_error": 0.026219767140527414 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0003454502088859115, 0.003228582704851672 ], "confidence_level": 0.95, "mean": 0.0016438518951055444, "replicates": 8, "sample_sd": 0.002320035913634523, "standard_error": 0.0008202565635636492 }, "log_loss": { "bootstrap_interval": [ 0.0013480018616257514, 0.08541421985403558 ], "confidence_level": 0.95, "mean": 0.04038284190561101, "replicates": 8, "sample_sd": 0.06657812250324625, "standard_error": 0.02353892095035705 } } }, "discount": { "bootstrap_interval": [ 0.15514966535683344, 0.6785090625782595 ], "confidence_level": 0.95, "mean": 0.37531609161541424, "replicates": 8, "sample_sd": 0.38758930289897636, "standard_error": 0.13703351219761647 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.08940819025149738, "discount_one_count": 2, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06362187818281137, 0.066449033322842 ], "confidence_level": 0.95, "mean": 0.06515315088115874, "replicates": 8, "sample_sd": 0.002239102956227824, "standard_error": 0.0007916424420617698 }, "log_loss": { "bootstrap_interval": [ 0.26324307319072754, 0.27117432366828315 ], "confidence_level": 0.95, "mean": 0.2674366675576201, "replicates": 8, "sample_sd": 0.006129304252645061, "standard_error": 0.002167036300500433 } }, "independent": { "brier": { "bootstrap_interval": [ 0.023865773180187372, 0.02705852085167304 ], "confidence_level": 0.95, "mean": 0.025535985286578865, "replicates": 8, "sample_sd": 0.0025093721501477464, "standard_error": 0.0008871970319450694 }, "log_loss": { "bootstrap_interval": [ 0.21041304603363456, 0.25366991271662814 ], "confidence_level": 0.95, "mean": 0.23064301537861712, "replicates": 8, "sample_sd": 0.033837003559132404, "standard_error": 0.011963187335847933 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11072065688134883, 0.11660060195503852 ], "confidence_level": 0.95, "mean": 0.1136443487378146, "replicates": 8, "sample_sd": 0.004602628198623606, "standard_error": 0.0016272748052635878 }, "log_loss": { "bootstrap_interval": [ 0.3509054274223263, 0.36740702473192133 ], "confidence_level": 0.95, "mean": 0.3590003481495725, "replicates": 8, "sample_sd": 0.0128353060091697, "standard_error": 0.004537965958844168 } }, "learned": { "brier": { "bootstrap_interval": [ 0.02017949748058727, 0.024996579683586057 ], "confidence_level": 0.95, "mean": 0.022565689363940204, "replicates": 8, "sample_sd": 0.003746300208879866, "standard_error": 0.001324517141029766 }, "log_loss": { "bootstrap_interval": [ 0.07626865609850368, 0.16053066941346725 ], "confidence_level": 0.95, "mean": 0.11450577878307815, "replicates": 8, "sample_sd": 0.06605682684777345, "standard_error": 0.023354615103863097 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.019436638925985135, 0.0224627640285008 ], "confidence_level": 0.95, "mean": 0.020921837468834657, "replicates": 8, "sample_sd": 0.002322164212277415, "standard_error": 0.0008210090307650388 }, "log_loss": { "bootstrap_interval": [ 0.07010157326191868, 0.07834704584925505 ], "confidence_level": 0.95, "mean": 0.07412293687746714, "replicates": 8, "sample_sd": 0.006422424149699534, "standard_error": 0.0022706698339543933 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.17391304347826086 ], "oracle_method": "oracle", "population_optimal_scalar": 0.17391304347826086, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.5-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.013830737336042044, -0.01033764528816408 ], "confidence_level": 0.95, "mean": -0.012097850870430804, "replicates": 8, "sample_sd": 0.002744601153855001, "standard_error": 0.0009703630437716469 }, "log_loss": { "bootstrap_interval": [ -0.04080279585875479, -0.01704883033781457 ], "confidence_level": 0.95, "mean": -0.030810110002396064, "replicates": 8, "sample_sd": 0.018938801502872887, "standard_error": 0.006695877485113698 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06453397209753307, -0.05261934461977784 ], "confidence_level": 0.95, "mean": -0.058889941518974274, "replicates": 8, "sample_sd": 0.00925089161583679, "standard_error": 0.003270684096789986 }, "log_loss": { "bootstrap_interval": [ -1.6424783516675956, -1.4447668582179354 ], "confidence_level": 0.95, "mean": -1.549268296104949, "replicates": 8, "sample_sd": 0.1518754402593199, "standard_error": 0.05369607685152874 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0009517861509683243, 0.0047421814251621805 ], "confidence_level": 0.95, "mean": 0.0026591555823511463, "replicates": 8, "sample_sd": 0.0029653504649403474, "standard_error": 0.0010484097111770005 }, "log_loss": { "bootstrap_interval": [ 0.003971603735093254, 0.02874997514103307 ], "confidence_level": 0.95, "mean": 0.01377821236579957, "replicates": 8, "sample_sd": 0.020359991903215004, "standard_error": 0.007198344169833265 } } }, "discount": { "bootstrap_interval": [ 0.0860110533291552, 0.1392557772983071 ], "confidence_level": 0.95, "mean": 0.10995499984940035, "replicates": 8, "sample_sd": 0.041519378577215584, "standard_error": 0.014679317071300303 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.19652597216104972, "discount_min": 0.06343887736676815, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17517890897331595, 0.18143707136173123 ], "confidence_level": 0.95, "mean": 0.1784462394918821, "replicates": 8, "sample_sd": 0.0047887274132715355, "standard_error": 0.0016930708135891085 }, "log_loss": { "bootstrap_interval": [ 0.5320755538618068, 0.5451703869365634 ], "confidence_level": 0.95, "mean": 0.5387780077454056, "replicates": 8, "sample_sd": 0.010318071397912418, "standard_error": 0.0036479891271154153 } }, "independent": { "brier": { "bootstrap_interval": [ 0.21609994432040963, 0.23296742193215086 ], "confidence_level": 0.95, "mean": 0.22523833014042557, "replicates": 8, "sample_sd": 0.013023763326512667, "standard_error": 0.004604595682372887 }, "log_loss": { "bootstrap_interval": [ 1.9483053701171245, 2.1515381491505106 ], "confidence_level": 0.95, "mean": 2.0572361938479586, "replicates": 8, "sample_sd": 0.15832719637851006, "standard_error": 0.05597711710274932 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19533711703824966, 0.20356075558717276 ], "confidence_level": 0.95, "mean": 0.19920056907551684, "replicates": 8, "sample_sd": 0.00638457994517526, "standard_error": 0.002257289887130531 }, "log_loss": { "bootstrap_interval": [ 0.5738972438996497, 0.5917867091840202 ], "confidence_level": 0.95, "mean": 0.5821224059622203, "replicates": 8, "sample_sd": 0.01413800323726898, "standard_error": 0.0049985389807551284 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1616716171215334, 0.17061635744707956 ], "confidence_level": 0.95, "mean": 0.1663483886214513, "replicates": 8, "sample_sd": 0.0070044556558761985, "standard_error": 0.002476449046395263 }, "log_loss": { "bootstrap_interval": [ 0.4928036625624296, 0.5243647843412086 ], "confidence_level": 0.95, "mean": 0.5079678977430095, "replicates": 8, "sample_sd": 0.024574200072181546, "standard_error": 0.008688291756637257 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.15874696352359566, 0.1681529041274286 ], "confidence_level": 0.95, "mean": 0.16368923303910016, "replicates": 8, "sample_sd": 0.007391043816413402, "standard_error": 0.0026131286013164083 }, "log_loss": { "bootstrap_interval": [ 0.4820900958099912, 0.505060042840387 ], "confidence_level": 0.95, "mean": 0.49418968537721, "replicates": 8, "sample_sd": 0.01758092433794739, "standard_error": 0.006215795409445106 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.09523809523809523 ], "oracle_method": "oracle", "population_optimal_scalar": 0.09523809523809523, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.5-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.020692709962080367, -0.013709342331758148 ], "confidence_level": 0.95, "mean": -0.017610576184650485, "replicates": 8, "sample_sd": 0.005593820806345021, "standard_error": 0.0019777143124544826 }, "log_loss": { "bootstrap_interval": [ -0.08811151396411772, 0.21028506226781418 ], "confidence_level": 0.95, "mean": 0.014497583127256113, "replicates": 8, "sample_sd": 0.2744987805543911, "standard_error": 0.09704997457872397 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.014045370337147289, -0.007174253614295792 ], "confidence_level": 0.95, "mean": -0.01095302041992521, "replicates": 8, "sample_sd": 0.005349213769649619, "standard_error": 0.0018912326652678498 }, "log_loss": { "bootstrap_interval": [ -0.8632123403181328, -0.5042084571333719 ], "confidence_level": 0.95, "mean": -0.7230069585923475, "replicates": 8, "sample_sd": 0.3009463585221502, "standard_error": 0.10640060544220516 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0001386958076151185, 0.00627540913801825 ], "confidence_level": 0.95, "mean": 0.002539042636998834, "replicates": 8, "sample_sd": 0.005231186197683089, "standard_error": 0.0018495036170155918 }, "log_loss": { "bootstrap_interval": [ 0.0020391720045691104, 0.3000699174895967 ], "confidence_level": 0.95, "mean": 0.10337812336234262, "replicates": 8, "sample_sd": 0.27688930621901425, "standard_error": 0.09789515303275173 } } }, "discount": { "bootstrap_interval": [ 0.088573812506381, 0.44293939865714316 ], "confidence_level": 0.95, "mean": 0.21323838932224726, "replicates": 8, "sample_sd": 0.3187688450964607, "standard_error": 0.11270180599935574 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.05966238249617803, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07568891569718139, 0.08051060631961854 ], "confidence_level": 0.95, "mean": 0.0782935504034101, "replicates": 8, "sample_sd": 0.003769200158674396, "standard_error": 0.001332613495924038 }, "log_loss": { "bootstrap_interval": [ 0.27777440068882975, 0.28980959107429954 ], "confidence_level": 0.95, "mean": 0.2840745183757948, "replicates": 8, "sample_sd": 0.009418915557316733, "standard_error": 0.0033300895310010652 } }, "independent": { "brier": { "bootstrap_interval": [ 0.06604650722200908, 0.07604542569915264 ], "confidence_level": 0.95, "mean": 0.07163599463868482, "replicates": 8, "sample_sd": 0.0077363346888963255, "standard_error": 0.0027352073600236553 }, "log_loss": { "bootstrap_interval": [ 0.9625672659296854, 1.0772800584726447 ], "confidence_level": 0.95, "mean": 1.0215790600953985, "replicates": 8, "sample_sd": 0.08816444964302679, "standard_error": 0.031170840101082065 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10871723167377877, 0.11587381268116806 ], "confidence_level": 0.95, "mean": 0.11230572728599801, "replicates": 8, "sample_sd": 0.005498152457137541, "standard_error": 0.0019438904432197166 }, "log_loss": { "bootstrap_interval": [ 0.34669863390951583, 0.36522337678971506 ], "confidence_level": 0.95, "mean": 0.3558041579520046, "replicates": 8, "sample_sd": 0.014597153738659536, "standard_error": 0.005160873197314361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.055850009726775285, 0.0651253152373707 ], "confidence_level": 0.95, "mean": 0.06068297421875961, "replicates": 8, "sample_sd": 0.007085539545858864, "standard_error": 0.0025051165306211264 }, "log_loss": { "bootstrap_interval": [ 0.1934804820876403, 0.49490951706134295 ], "confidence_level": 0.95, "mean": 0.2985721015030509, "replicates": 8, "sample_sd": 0.2736586089559874, "standard_error": 0.09675292906142818 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.054635622018023335, 0.0610855299259089 ], "confidence_level": 0.95, "mean": 0.058143931581760774, "replicates": 8, "sample_sd": 0.005020865586578262, "standard_error": 0.0017751440518478307 }, "log_loss": { "bootstrap_interval": [ 0.18626412217685512, 0.2028898455435619 ], "confidence_level": 0.95, "mean": 0.1951939781407083, "replicates": 8, "sample_sd": 0.012851618354273401, "standard_error": 0.00454373324376411 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.09523809523809523 ], "oracle_method": "oracle", "population_optimal_scalar": 0.09523809523809523, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.75-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0016578133636269807, 0.0006981040770977116 ], "confidence_level": 0.95, "mean": -0.00041009729265563746, "replicates": 8, "sample_sd": 0.0018502277628306973, "standard_error": 0.0006541542989186006 }, "log_loss": { "bootstrap_interval": [ -0.005086587200569079, 0.006460329299863506 ], "confidence_level": 0.95, "mean": 0.00035121892373074504, "replicates": 8, "sample_sd": 0.00882569738036462, "standard_error": 0.0031203552331780854 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.08451818993916459, -0.07070914549857812 ], "confidence_level": 0.95, "mean": -0.07817851302895004, "replicates": 8, "sample_sd": 0.01089688684352208, "standard_error": 0.0038526312904384676 }, "log_loss": { "bootstrap_interval": [ -2.5785739580403124, -2.3018249661395314 ], "confidence_level": 0.95, "mean": -2.4520765891885827, "replicates": 8, "sample_sd": 0.2136506211381417, "standard_error": 0.07553690150574896 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010584024804339764, 0.00448124295910608 ], "confidence_level": 0.95, "mean": 0.002595375579997849, "replicates": 8, "sample_sd": 0.002685498217858197, "standard_error": 0.0009494670003559596 }, "log_loss": { "bootstrap_interval": [ 0.003678706940303929, 0.01730964482629408 ], "confidence_level": 0.95, "mean": 0.009820880487749942, "replicates": 8, "sample_sd": 0.010468895527609668, "standard_error": 0.003701313509553158 } } }, "discount": { "bootstrap_interval": [ 0.05819800097632473, 0.08835425963670351 ], "confidence_level": 0.95, "mean": 0.07259963890300981, "replicates": 8, "sample_sd": 0.023447442792609303, "standard_error": 0.008289922900068839 }, "discount_below_exchangeable_range_count": 2, "discount_max": 0.11525086922728961, "discount_min": 0.04443839954001484, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1839992202574754, 0.1917990398935091 ], "confidence_level": 0.95, "mean": 0.18808317069152167, "replicates": 8, "sample_sd": 0.0060998163901216235, "standard_error": 0.0021566107667239233 }, "log_loss": { "bootstrap_interval": [ 0.5485380838096635, 0.5655209250685318 ], "confidence_level": 0.95, "mean": 0.5573548345745812, "replicates": 8, "sample_sd": 0.013228841794008944, "standard_error": 0.004677101869893868 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2559960913944258, 0.27421581418101204 ], "confidence_level": 0.95, "mean": 0.2658515864278161, "replicates": 8, "sample_sd": 0.014434583368743785, "standard_error": 0.005103395891820644 }, "log_loss": { "bootstrap_interval": [ 2.8568093059068915, 3.1400181834474203 ], "confidence_level": 0.95, "mean": 3.0097826426868948, "replicates": 8, "sample_sd": 0.2212833935046529, "standard_error": 0.07823549405555563 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19419609588866374, 0.20340605236337786 ], "confidence_level": 0.95, "mean": 0.19869529628463112, "replicates": 8, "sample_sd": 0.007071188646351022, "standard_error": 0.0025000427214420654 }, "log_loss": { "bootstrap_interval": [ 0.5713346379186712, 0.5909918924117167 ], "confidence_level": 0.95, "mean": 0.58080379894007, "replicates": 8, "sample_sd": 0.015418508102089555, "standard_error": 0.005451265817383624 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18340692829754618, 0.1916715698993657 ], "confidence_level": 0.95, "mean": 0.18767307339886605, "replicates": 8, "sample_sd": 0.006554334919208594, "standard_error": 0.0023173073337700895 }, "log_loss": { "bootstrap_interval": [ 0.5462423287836506, 0.5682862975675257 ], "confidence_level": 0.95, "mean": 0.557706053498312, "replicates": 8, "sample_sd": 0.017019667264309965, "standard_error": 0.006017361068066136 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.17999915855848847, 0.18959364287760427 ], "confidence_level": 0.95, "mean": 0.18507769781886818, "replicates": 8, "sample_sd": 0.0074958251425714835, "standard_error": 0.0026501743944504575 }, "log_loss": { "bootstrap_interval": [ 0.5362932942481862, 0.5584102289648908 ], "confidence_level": 0.95, "mean": 0.547885173010562, "replicates": 8, "sample_sd": 0.017021513062078063, "standard_error": 0.006018013656125396 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.06557377049180328 ], "oracle_method": "oracle", "population_optimal_scalar": 0.06557377049180328, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho0.75-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.005202082938242707, -0.0002508107120945801 ], "confidence_level": 0.95, "mean": -0.0030806475595214003, "replicates": 8, "sample_sd": 0.0038618261842788084, "standard_error": 0.0013653617413336575 }, "log_loss": { "bootstrap_interval": [ -0.021136939928217267, 0.0318411699737093 ], "confidence_level": 0.95, "mean": -0.0015986150379536032, "replicates": 8, "sample_sd": 0.046418531712208125, "standard_error": 0.016411429273212583 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.030677996161192496, -0.019328505825309712 ], "confidence_level": 0.95, "mean": -0.025149389247791247, "replicates": 8, "sample_sd": 0.008810040814026187, "standard_error": 0.0031148198010640836 }, "log_loss": { "bootstrap_interval": [ -1.847189816222371, -1.6049858437340279 ], "confidence_level": 0.95, "mean": -1.735534649035197, "replicates": 8, "sample_sd": 0.18560977225025768, "standard_error": 0.06562296430632393 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0004628628829783602, 0.004914982122301979 ], "confidence_level": 0.95, "mean": 0.002224514446880701, "replicates": 8, "sample_sd": 0.003664030066894095, "standard_error": 0.0012954302533861068 }, "log_loss": { "bootstrap_interval": [ 0.003311703599558319, 0.057167777030667825 ], "confidence_level": 0.95, "mean": 0.02263925362852178, "replicates": 8, "sample_sd": 0.047834877870627394, "standard_error": 0.016912183259775473 } } }, "discount": { "bootstrap_interval": [ 0.0606912105197398, 0.11632125713026475 ], "confidence_level": 0.95, "mean": 0.08359226957466191, "replicates": 8, "sample_sd": 0.045294083593426064, "standard_error": 0.01601387682827096 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.18937579846061708, "discount_min": 0.04377134493593053, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08911757045795149, 0.0961690067671329 ], "confidence_level": 0.95, "mean": 0.09297562420353894, "replicates": 8, "sample_sd": 0.005487210419030562, "standard_error": 0.0019400218485469936 }, "log_loss": { "bootstrap_interval": [ 0.300371885183796, 0.31713458491900276 ], "confidence_level": 0.95, "mean": 0.30915787100330544, "replicates": 8, "sample_sd": 0.013028964419553582, "standard_error": 0.0046064345464522935 } }, "independent": { "brier": { "bootstrap_interval": [ 0.10523341524627858, 0.12361619463258346 ], "confidence_level": 0.95, "mean": 0.11504436589180879, "replicates": 8, "sample_sd": 0.014181342977279205, "standard_error": 0.005013861892783174 }, "log_loss": { "bootstrap_interval": [ 1.9142546655833907, 2.1515964183050693 ], "confidence_level": 0.95, "mean": 2.043093905000549, "replicates": 8, "sample_sd": 0.1857206209772257, "standard_error": 0.06566215524958642 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10693496895891191, 0.11526578830451026 ], "confidence_level": 0.95, "mean": 0.11121122109872107, "replicates": 8, "sample_sd": 0.006408123641965544, "standard_error": 0.002265613840957836 }, "log_loss": { "bootstrap_interval": [ 0.34250853784567753, 0.3629722217186941 ], "confidence_level": 0.95, "mean": 0.3526925037658831, "replicates": 8, "sample_sd": 0.01593143496684411, "standard_error": 0.005632612849543975 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0847952067192303, 0.09406604223822881 ], "confidence_level": 0.95, "mean": 0.08989497664401755, "replicates": 8, "sample_sd": 0.0071520531195012485, "standard_error": 0.002528632630102867 }, "log_loss": { "bootstrap_interval": [ 0.28304660573535545, 0.341959942011886 ], "confidence_level": 0.95, "mean": 0.30755925596535183, "replicates": 8, "sample_sd": 0.04706124928410642, "standard_error": 0.0166386642499511 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.0829255592144923, 0.09162458550558766 ], "confidence_level": 0.95, "mean": 0.08767046219713684, "replicates": 8, "sample_sd": 0.006660333036460096, "standard_error": 0.002354783327520861 }, "log_loss": { "bootstrap_interval": [ 0.27330606337162683, 0.29445712400481616 ], "confidence_level": 0.95, "mean": 0.28492000233683, "replicates": 8, "sample_sd": 0.01640003620653811, "standard_error": 0.0057982884066739996 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.06557377049180328 ], "oracle_method": "oracle", "population_optimal_scalar": 0.06557377049180328, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho1-a0.5": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.001055761515227198, 0.004995162256541733 ], "confidence_level": 0.95, "mean": 0.0026696714486234216, "replicates": 8, "sample_sd": 0.0032450327229714957, "standard_error": 0.0011472923217926958 }, "log_loss": { "bootstrap_interval": [ 0.003526760299888826, 0.014217586734715171 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.09884514120778132, -0.08198099521357964 ], "confidence_level": 0.95, "mean": -0.09123989330445748, "replicates": 8, "sample_sd": 0.013211988013826007, "standard_error": 0.004671143158765877 }, "log_loss": { "bootstrap_interval": [ -3.44272431646371, -3.1044113720677986 ], "confidence_level": 0.95, "mean": -3.2880999524515966, "replicates": 8, "sample_sd": 0.26377517326691463, "standard_error": 0.09325860686284593 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.001055761515227198, 0.004995162256541733 ], "confidence_level": 0.95, "mean": 0.0026696714486234216, "replicates": 8, "sample_sd": 0.0032450327229714957, "standard_error": 0.0011472923217926958 }, "log_loss": { "bootstrap_interval": [ 0.003526760299888826, 0.014217586734715171 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } } }, "discount": { "bootstrap_interval": [ 0.04355972990184746, 0.06371163698291328 ], "confidence_level": 0.95, "mean": 0.053846042928676105, "replicates": 8, "sample_sd": 0.01576705346376195, "standard_error": 0.005574495211778458 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.07592377547385129, "discount_min": 0.030747155079197702, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19303102399585964, 0.20254757552554037 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5684267353075054, 0.5891079960061936 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "independent": { "brier": { "bootstrap_interval": [ 0.28046367675438383, 0.3017638417577789 ], "confidence_level": 0.95, "mean": 0.29199312853791404, "replicates": 8, "sample_sd": 0.016574657964399744, "standard_error": 0.005860026521237338 }, "log_loss": { "bootstrap_interval": [ 3.6865163594404415, 4.036218042686435 ], "confidence_level": 0.95, "mean": 3.8755849984546744, "replicates": 8, "sample_sd": 0.27155088585626547, "standard_error": 0.09600773641308971 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19312639538623413, 0.20262246067179612 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5683011560395523, 0.5894489306743499 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1965772814757764, 0.20483046110547987 ], "confidence_level": 0.95, "mean": 0.20075323523345656, "replicates": 8, "sample_sd": 0.006395098492831272, "standard_error": 0.002261008755318431 }, "log_loss": { "bootstrap_interval": [ 0.5775688078035017, 0.5971347139846432 ], "confidence_level": 0.95, "mean": 0.5874850460030775, "replicates": 8, "sample_sd": 0.015094497522713133, "standard_error": 0.005336710778456999 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19309124135398428, 0.2026225621140726 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5685327747033705, 0.5893518842277922 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.05 ], "oracle_method": "oracle", "population_optimal_scalar": 0.05, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n20-rho1-a1": { "condition": { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0008023291901622548, 0.003416482756609874 ], "confidence_level": 0.95, "mean": 0.002033601459410844, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.004676454963753737, 0.020129946049220603 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922939, "standard_error": 0.004312314006328675 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04011108941594338, -0.031143623273019394 ], "confidence_level": 0.95, "mean": -0.03590156983638039, "replicates": 8, "sample_sd": 0.006935430235959215, "standard_error": 0.002452044875146489 }, "log_loss": { "bootstrap_interval": [ -2.9546309832124575, -2.6178993023323693 ], "confidence_level": 0.95, "mean": -2.79812768209896, "replicates": 8, "sample_sd": 0.26343013498522483, "standard_error": 0.09313661740847001 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0008023291901622548, 0.003416482756609874 ], "confidence_level": 0.95, "mean": 0.002033601459410844, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.004676454963753737, 0.020129946049220603 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922939, "standard_error": 0.004312314006328675 } } }, "discount": { "bootstrap_interval": [ 0.04528735743474995, 0.06868676578575182 ], "confidence_level": 0.95, "mean": 0.05649055321917837, "replicates": 8, "sample_sd": 0.018320612302033905, "standard_error": 0.006477314597128929 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.09060705289228149, "discount_min": 0.035255870930271466, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 20 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.10476898350202013, 0.11418217719281068 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.3368229045995601, 0.3599549011956061 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "independent": { "brier": { "bootstrap_interval": [ 0.13858931971235716, 0.1556525967658685 ], "confidence_level": 0.95, "mean": 0.1478468137186287, "replicates": 8, "sample_sd": 0.013410234562152809, "standard_error": 0.004741233898100231 }, "log_loss": { "bootstrap_interval": [ 2.9714096792474245, 3.3198767922189982 ], "confidence_level": 0.95, "mean": 3.1589987612763775, "replicates": 8, "sample_sd": 0.2758504963117007, "standard_error": 0.09752787826783914 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10471645411705244, 0.11423989557542283 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.33705525707635287, 0.3600592149920769 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10732907846090697, 0.11602566352172425 ], "confidence_level": 0.95, "mean": 0.1119452438822483, "replicates": 8, "sample_sd": 0.006750192655061165, "standard_error": 0.002386553500354688 }, "log_loss": { "bootstrap_interval": [ 0.3489391264948322, 0.37193955080059843 ], "confidence_level": 0.95, "mean": 0.3608710791774176, "replicates": 8, "sample_sd": 0.01801647685394999, "standard_error": 0.0063697864782592565 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1047474530089553, 0.11423989557542283 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.33700815210545776, 0.3602404396258791 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.05 ], "oracle_method": "oracle", "population_optimal_scalar": 0.05, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07365666767490184, -0.0704076552753146 ], "confidence_level": 0.95, "mean": -0.07220320305843274, "replicates": 8, "sample_sd": 0.0025493544109616638, "standard_error": 0.0009013328958194144 }, "log_loss": { "bootstrap_interval": [ -0.22343666796854364, -0.20907742626907244 ], "confidence_level": 0.95, "mean": -0.21671527447425773, "replicates": 8, "sample_sd": 0.011219636641130142, "standard_error": 0.0039667405756960905 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 1.948897973772651e-05, 0.0006681371519771627 ], "confidence_level": 0.95, "mean": 0.0002929346683839089, "replicates": 8, "sample_sd": 0.0004904616483020787, "standard_error": 0.00017340437871316567 }, "log_loss": { "bootstrap_interval": [ 0.00026649291285194693, 0.003961602435479501 ], "confidence_level": 0.95, "mean": 0.0018684779250356456, "replicates": 8, "sample_sd": 0.002781210630128596, "standard_error": 0.0009833064482360205 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 1.948897973772651e-05, 0.0006681371519771627 ], "confidence_level": 0.95, "mean": 0.0002929346683839089, "replicates": 8, "sample_sd": 0.0004904616483020787, "standard_error": 0.00017340437871316567 }, "log_loss": { "bootstrap_interval": [ 0.00026649291285194693, 0.003961602435479501 ], "confidence_level": 0.95, "mean": 0.0018684779250356456, "replicates": 8, "sample_sd": 0.002781210630128596, "standard_error": 0.0009833064482360205 } } }, "discount": { "bootstrap_interval": [ 0.8656124081223328, 0.9825116294217536 ], "confidence_level": 0.95, "mean": 0.9286152698830085, "replicates": 8, "sample_sd": 0.09299397750944101, "standard_error": 0.03287833605321751 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7554341497409709, "discount_one_count": 3, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16443675602680188, 0.16838703584235407 ], "confidence_level": 0.95, "mean": 0.16644976262976788, "replicates": 8, "sample_sd": 0.003080671258908512, "standard_error": 0.0010891817688903533 }, "log_loss": { "bootstrap_interval": [ 0.5147549633906062, 0.5233720631812387 ], "confidence_level": 0.95, "mean": 0.5191519574313715, "replicates": 8, "sample_sd": 0.006689767225118888, "standard_error": 0.002365189884720539 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0910339039577768, 0.09674577440303517 ], "confidence_level": 0.95, "mean": 0.09395362490295123, "replicates": 8, "sample_sd": 0.004412940789109789, "standard_error": 0.001560210178477123 }, "log_loss": { "bootstrap_interval": [ 0.29123802011174016, 0.3094443875005619 ], "confidence_level": 0.95, "mean": 0.30056820503207815, "replicates": 8, "sample_sd": 0.014201896342561478, "standard_error": 0.005021128604766824 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1976170632399488, 0.20331341147825113 ], "confidence_level": 0.95, "mean": 0.200401419084981, "replicates": 8, "sample_sd": 0.004402965745277611, "standard_error": 0.00155668346790894 }, "log_loss": { "bootstrap_interval": [ 0.5786046854530827, 0.5916353464605317 ], "confidence_level": 0.95, "mean": 0.5850450001139459, "replicates": 8, "sample_sd": 0.01030599781392939, "standard_error": 0.0036437204705616027 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09116986714526973, 0.09722246635756823 ], "confidence_level": 0.95, "mean": 0.09424655957133513, "replicates": 8, "sample_sd": 0.004661175412978492, "standard_error": 0.0016479743714085486 }, "log_loss": { "bootstrap_interval": [ 0.2920355176566837, 0.31228319253074177 ], "confidence_level": 0.95, "mean": 0.3024366829571138, "replicates": 8, "sample_sd": 0.015770067291577202, "standard_error": 0.005575560760821205 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.09104693772806449, 0.09671197171220713 ], "confidence_level": 0.95, "mean": 0.09395362490295123, "replicates": 8, "sample_sd": 0.004412940789109789, "standard_error": 0.001560210178477123 }, "log_loss": { "bootstrap_interval": [ 0.2910194532786355, 0.3096103479478619 ], "confidence_level": 0.95, "mean": 0.30056820503207815, "replicates": 8, "sample_sd": 0.014201896342561478, "standard_error": 0.005021128604766824 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05551856346544885, -0.05303151323366071 ], "confidence_level": 0.95, "mean": -0.05424996896042778, "replicates": 8, "sample_sd": 0.0019153201247607476, "standard_error": 0.0006771679241806944 }, "log_loss": { "bootstrap_interval": [ -0.24053116630605176, -0.23208011431547837 ], "confidence_level": 0.95, "mean": -0.2363760464751835, "replicates": 8, "sample_sd": 0.006550475298967836, "standard_error": 0.002315942751947567 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 1.7812183072040896e-06 ], "confidence_level": 0.95, "mean": 7.479466148352897e-07, "replicates": 8, "sample_sd": 1.513336527736444e-06, "standard_error": 5.350452604898716e-07 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0005297878132664636 ], "confidence_level": 0.95, "mean": 0.00018051523759340634, "replicates": 8, "sample_sd": 0.0004918459801118119, "standard_error": 0.000173893813918203 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0, 1.7812183072040896e-06 ], "confidence_level": 0.95, "mean": 7.479466148352897e-07, "replicates": 8, "sample_sd": 1.513336527736444e-06, "standard_error": 5.350452604898716e-07 }, "log_loss": { "bootstrap_interval": [ 0.0, 0.0005297878132664636 ], "confidence_level": 0.95, "mean": 0.00018051523759340634, "replicates": 8, "sample_sd": 0.0004918459801118119, "standard_error": 0.000173893813918203 } } }, "discount": { "bootstrap_interval": [ 0.9503972927434791, 1.0 ], "confidence_level": 0.95, "mean": 0.9829985765857141, "replicates": 8, "sample_sd": 0.04586399491442324, "standard_error": 0.016215370908147 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8695948646310598, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06123640299972895, 0.06371512459834042 ], "confidence_level": 0.95, "mean": 0.06245304283975689, "replicates": 8, "sample_sd": 0.0019040820829993007, "standard_error": 0.0006731946764123061 }, "log_loss": { "bootstrap_interval": [ 0.26162424189678346, 0.26858853407876404 ], "confidence_level": 0.95, "mean": 0.2651159615525675, "replicates": 8, "sample_sd": 0.005394904952398857, "standard_error": 0.0019073869378490598 } }, "independent": { "brier": { "bootstrap_interval": [ 0.007187136066321112, 0.009317331572403966 ], "confidence_level": 0.95, "mean": 0.008202325932714277, "replicates": 8, "sample_sd": 0.0016448977035854442, "standard_error": 0.0005815591602817236 }, "log_loss": { "bootstrap_interval": [ 0.025317878360984813, 0.03220012691671342 ], "confidence_level": 0.95, "mean": 0.028559399839790566, "replicates": 8, "sample_sd": 0.005308601339830792, "standard_error": 0.0018768740030051722 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1121214803504369, 0.11751135474811658 ], "confidence_level": 0.95, "mean": 0.11474074946836885, "replicates": 8, "sample_sd": 0.004152437295533535, "standard_error": 0.0014681082850618453 }, "log_loss": { "bootstrap_interval": [ 0.35511518442884876, 0.3701690053034555 ], "confidence_level": 0.95, "mean": 0.36231777974411405, "replicates": 8, "sample_sd": 0.011648816374946308, "standard_error": 0.004118478525760715 } }, "learned": { "brier": { "bootstrap_interval": [ 0.007169357922995625, 0.009278431711096193 ], "confidence_level": 0.95, "mean": 0.008203073879329113, "replicates": 8, "sample_sd": 0.0016448530267195972, "standard_error": 0.0005815433646243223 }, "log_loss": { "bootstrap_interval": [ 0.0254444620875864, 0.032216786249094444 ], "confidence_level": 0.95, "mean": 0.028739915077383972, "replicates": 8, "sample_sd": 0.005223946885774941, "standard_error": 0.0018469441337449037 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.007193357720770152, 0.009291348880739585 ], "confidence_level": 0.95, "mean": 0.008202325932714277, "replicates": 8, "sample_sd": 0.0016448977035854442, "standard_error": 0.0005815591602817236 }, "log_loss": { "bootstrap_interval": [ 0.02525620076491278, 0.03211620150221299 ], "confidence_level": 0.95, "mean": 0.028559399839790566, "replicates": 8, "sample_sd": 0.005308601339830792, "standard_error": 0.0018768740030051722 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0 ], "oracle_method": "oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.1-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.049634939318614686, -0.04668375667814046 ], "confidence_level": 0.95, "mean": -0.048217387814470554, "replicates": 8, "sample_sd": 0.002288725812838571, "standard_error": 0.0008091867712674232 }, "log_loss": { "bootstrap_interval": [ -0.14489880916112607, -0.13607313816376485 ], "confidence_level": 0.95, "mean": -0.14062295227712884, "replicates": 8, "sample_sd": 0.0068421944690213055, "standard_error": 0.002419081053621027 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0021091663368101848, 0.0001943375923777156 ], "confidence_level": 0.95, "mean": -0.0009488371892219169, "replicates": 8, "sample_sd": 0.0017673347748599676, "standard_error": 0.0006248472019651416 }, "log_loss": { "bootstrap_interval": [ -0.011584285075919937, -0.0012828306676187615 ], "confidence_level": 0.95, "mean": -0.006761582386519825, "replicates": 8, "sample_sd": 0.008055019967427586, "standard_error": 0.0028478796207805444 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0005600074681862618, 0.0021697422943334314 ], "confidence_level": 0.95, "mean": 0.0013185512030580247, "replicates": 8, "sample_sd": 0.0012577937846301266, "standard_error": 0.00044469725722312716 }, "log_loss": { "bootstrap_interval": [ 0.0029741787404199763, 0.01080314613880781 ], "confidence_level": 0.95, "mean": 0.0067629139535961, "replicates": 8, "sample_sd": 0.006090217345946454, "standard_error": 0.0021532169921093377 } } }, "discount": { "bootstrap_interval": [ 0.5956463951122171, 0.8288127058985009 ], "confidence_level": 0.95, "mean": 0.7064608352268326, "replicates": 8, "sample_sd": 0.18103402490354004, "standard_error": 0.06400519331739374 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.5235146268137592, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16734742256575796, 0.1717400100811583 ], "confidence_level": 0.95, "mean": 0.16953419964286, "replicates": 8, "sample_sd": 0.003406188848587928, "standard_error": 0.001204269616419261 }, "log_loss": { "bootstrap_interval": [ 0.5192394360690981, 0.5289038925936813 ], "confidence_level": 0.95, "mean": 0.5239863926828218, "replicates": 8, "sample_sd": 0.007465113688811839, "standard_error": 0.0026393162558436866 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1189150458687535, 0.12528551836465918 ], "confidence_level": 0.95, "mean": 0.12226564901761137, "replicates": 8, "sample_sd": 0.00496175281934619, "standard_error": 0.0017542445325655807 }, "log_loss": { "bootstrap_interval": [ 0.3802031960485411, 0.39901368330854214 ], "confidence_level": 0.95, "mean": 0.3901250227922128, "replicates": 8, "sample_sd": 0.01462204029767262, "standard_error": 0.005169671924633637 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19725727672058868, 0.20350967701860387 ], "confidence_level": 0.95, "mean": 0.20005045972324384, "replicates": 8, "sample_sd": 0.004854666888269692, "standard_error": 0.0017163839385486471 }, "log_loss": { "bootstrap_interval": [ 0.577903048029692, 0.5919409679860252 ], "confidence_level": 0.95, "mean": 0.5842897509669385, "replicates": 8, "sample_sd": 0.01106838076050973, "standard_error": 0.003913263546255573 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11852405129141545, 0.1241904962873805 ], "confidence_level": 0.95, "mean": 0.12131681182838946, "replicates": 8, "sample_sd": 0.004400582990755908, "standard_error": 0.0015558410369688402 }, "log_loss": { "bootstrap_interval": [ 0.3769113237373421, 0.3907056983546965 ], "confidence_level": 0.95, "mean": 0.383363440405693, "replicates": 8, "sample_sd": 0.010811281172479582, "standard_error": 0.00382236511518738 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.11694951243068982, 0.1228899850721536 ], "confidence_level": 0.95, "mean": 0.11999826062533143, "replicates": 8, "sample_sd": 0.004632283947299337, "standard_error": 0.0016377596957584744 }, "log_loss": { "bootstrap_interval": [ 0.3680036576993478, 0.384566135254264 ], "confidence_level": 0.95, "mean": 0.37660052645209685, "replicates": 8, "sample_sd": 0.012656984239819013, "standard_error": 0.0044749196926736415 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.7142857142857143 ], "oracle_method": "oracle", "population_optimal_scalar": 0.7142857142857143, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.1-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04855470975182355, -0.043653318349963506 ], "confidence_level": 0.95, "mean": -0.046037505380837486, "replicates": 8, "sample_sd": 0.003790464085005069, "standard_error": 0.001340131429175573 }, "log_loss": { "bootstrap_interval": [ -0.20101346680366083, -0.1831562326551248 ], "confidence_level": 0.95, "mean": -0.1921881891713943, "replicates": 8, "sample_sd": 0.014088867113974184, "standard_error": 0.004981166737763644 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 8.808587108380915e-07, 0.0013247494176102498 ], "confidence_level": 0.95, "mean": 0.00045760590329443863, "replicates": 8, "sample_sd": 0.001218320807460833, "standard_error": 0.0004307414523081125 }, "log_loss": { "bootstrap_interval": [ -0.0014022867737552222, 0.009648504826451867 ], "confidence_level": 0.95, "mean": 0.0027372900586679827, "replicates": 8, "sample_sd": 0.0093992262946213, "standard_error": 0.0033231283254168136 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ -0.00023881864837042334, 0.0013952147997858303 ], "confidence_level": 0.95, "mean": 0.00042886174397885443, "replicates": 8, "sample_sd": 0.0013422351927946021, "standard_error": 0.00047455180338614806 }, "log_loss": { "bootstrap_interval": [ 0.0013885445694390244, 0.012863700240268592 ], "confidence_level": 0.95, "mean": 0.0061077481108422135, "replicates": 8, "sample_sd": 0.009138926763365618, "standard_error": 0.003231098543571527 } } }, "discount": { "bootstrap_interval": [ 0.7270146919060552, 0.9976800860737771 ], "confidence_level": 0.95, "mean": 0.8823150812467784, "replicates": 8, "sample_sd": 0.2270037073783066, "standard_error": 0.08025793042084366 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.38807809834199836, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.06491377277591177, 0.06763522955979215 ], "confidence_level": 0.95, "mean": 0.06632417773356607, "replicates": 8, "sample_sd": 0.002096432258758303, "standard_error": 0.0007412007332331135 }, "log_loss": { "bootstrap_interval": [ 0.26520058982366423, 0.2729723496868341 ], "confidence_level": 0.95, "mean": 0.2692223574316937, "replicates": 8, "sample_sd": 0.006001776649103748, "standard_error": 0.002121948483874167 } }, "independent": { "brier": { "bootstrap_interval": [ 0.017992653809257235, 0.02148658563588017 ], "confidence_level": 0.95, "mean": 0.01982906644943414, "replicates": 8, "sample_sd": 0.0027035676944609037, "standard_error": 0.0009558555250750925 }, "log_loss": { "bootstrap_interval": [ 0.06745446023336815, 0.0810377098337286 ], "confidence_level": 0.95, "mean": 0.07429687820163143, "replicates": 8, "sample_sd": 0.010502806293858845, "standard_error": 0.00371330277593817 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11172430854360972, 0.11721868779922734 ], "confidence_level": 0.95, "mean": 0.11440040546560537, "replicates": 8, "sample_sd": 0.004214884081710997, "standard_error": 0.0014901865580465399 }, "log_loss": { "bootstrap_interval": [ 0.3536139453181758, 0.368522173568717 ], "confidence_level": 0.95, "mean": 0.3609438993647717, "replicates": 8, "sample_sd": 0.011549219867090432, "standard_error": 0.00408326584271702 } }, "learned": { "brier": { "bootstrap_interval": [ 0.018483674479025772, 0.021821506153971326 ], "confidence_level": 0.95, "mean": 0.020286672352728578, "replicates": 8, "sample_sd": 0.0026134609607101444, "standard_error": 0.0009239979838422261 }, "log_loss": { "bootstrap_interval": [ 0.07004264771495404, 0.08368543800506476 ], "confidence_level": 0.95, "mean": 0.07703416826029942, "replicates": 8, "sample_sd": 0.01059602852834448, "standard_error": 0.0037462618130192473 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.01825342255288851, 0.02133327405158246 ], "confidence_level": 0.95, "mean": 0.019857810608749724, "replicates": 8, "sample_sd": 0.00240500115634337, "standard_error": 0.0008502963132059425 }, "log_loss": { "bootstrap_interval": [ 0.0660532933261412, 0.07571731282778636 ], "confidence_level": 0.95, "mean": 0.0709264201494572, "replicates": 8, "sample_sd": 0.007447251987303858, "standard_error": 0.002633001190713775 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.7142857142857143 ], "oracle_method": "oracle", "population_optimal_scalar": 0.7142857142857143, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.25-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.026714926592663012, -0.023836696453514387 ], "confidence_level": 0.95, "mean": -0.02526543105659524, "replicates": 8, "sample_sd": 0.0022536125061061, "standard_error": 0.0007967723426172165 }, "log_loss": { "bootstrap_interval": [ -0.0779610815965463, -0.05671804709993954 ], "confidence_level": 0.95, "mean": -0.06928147203273974, "replicates": 8, "sample_sd": 0.01757515520122575, "standard_error": 0.006213755711596374 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.010364531533262902, -0.00583508615518203 ], "confidence_level": 0.95, "mean": -0.00851668020953543, "replicates": 8, "sample_sd": 0.0036483952722820433, "standard_error": 0.0012899025187397865 }, "log_loss": { "bootstrap_interval": [ -0.06993338492067103, -0.039725549300369714 ], "confidence_level": 0.95, "mean": -0.05765121483830435, "replicates": 8, "sample_sd": 0.02469651172026632, "standard_error": 0.00873153545452668 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0008912815813983692, 0.004301361247691967 ], "confidence_level": 0.95, "mean": 0.002379927929669543, "replicates": 8, "sample_sd": 0.0026351137602957176, "standard_error": 0.0009316534045515422 }, "log_loss": { "bootstrap_interval": [ 0.003616376822114372, 0.028341014195620684 ], "confidence_level": 0.95, "mean": 0.013526785727427902, "replicates": 8, "sample_sd": 0.02013199384546526, "standard_error": 0.007117734683467163 } } }, "discount": { "bootstrap_interval": [ 0.4156142398731318, 0.6853639101908902 ], "confidence_level": 0.95, "mean": 0.5346884212864265, "replicates": 8, "sample_sd": 0.21401502396982852, "standard_error": 0.07566573736243362 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.35139327827070543, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17162994357592312, 0.17680659518308497 ], "confidence_level": 0.95, "mean": 0.17421503923827347, "replicates": 8, "sample_sd": 0.0040510735896588735, "standard_error": 0.0014322708031667593 }, "log_loss": { "bootstrap_interval": [ 0.5260978666605607, 0.5374866781020294 ], "confidence_level": 0.95, "mean": 0.5316619800684294, "replicates": 8, "sample_sd": 0.008857382731378064, "standard_error": 0.0031315576964610265 } }, "independent": { "brier": { "bootstrap_interval": [ 0.15337482072630038, 0.1616730018055658 ], "confidence_level": 0.95, "mean": 0.15746628839121368, "replicates": 8, "sample_sd": 0.006466952743365274, "standard_error": 0.0022864130692232657 }, "log_loss": { "bootstrap_interval": [ 0.5073741282286749, 0.5325003111323642 ], "confidence_level": 0.95, "mean": 0.5200317228739939, "replicates": 8, "sample_sd": 0.019460362964122763, "standard_error": 0.006880277308141373 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19651445611667873, 0.2034394148286022 ], "confidence_level": 0.95, "mean": 0.19971191477641081, "replicates": 8, "sample_sd": 0.005506869817303598, "standard_error": 0.001946972495463449 }, "log_loss": { "bootstrap_interval": [ 0.5764155152453672, 0.5923575842322727 ], "confidence_level": 0.95, "mean": 0.5834449207252018, "replicates": 8, "sample_sd": 0.012400593424502377, "standard_error": 0.004384271850601471 } }, "learned": { "brier": { "bootstrap_interval": [ 0.14569389763616708, 0.15229013526953142 ], "confidence_level": 0.95, "mean": 0.14894960818167824, "replicates": 8, "sample_sd": 0.005098693962857411, "standard_error": 0.0018026605381656928 }, "log_loss": { "bootstrap_interval": [ 0.45130259833608494, 0.4750720307893019 ], "confidence_level": 0.95, "mean": 0.4623805080356896, "replicates": 8, "sample_sd": 0.018746958855336837, "standard_error": 0.006628050866616937 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.1429170506565114, 0.15033066368346432 ], "confidence_level": 0.95, "mean": 0.1465696802520087, "replicates": 8, "sample_sd": 0.005728631346303798, "standard_error": 0.002025377035944618 }, "log_loss": { "bootstrap_interval": [ 0.43970359483083366, 0.4583280496959297 ], "confidence_level": 0.95, "mean": 0.4488537223082617, "replicates": 8, "sample_sd": 0.01462572510862245, "standard_error": 0.005170974702038644 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.25-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0340950396611349, -0.02974955639461989 ], "confidence_level": 0.95, "mean": -0.03194578519912143, "replicates": 8, "sample_sd": 0.003337294490401524, "standard_error": 0.0011799117824897105 }, "log_loss": { "bootstrap_interval": [ -0.13845344798389964, -0.11775690501862093 ], "confidence_level": 0.95, "mean": -0.12779026193513676, "replicates": 8, "sample_sd": 0.016243136150672518, "standard_error": 0.005742815859938445 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.001535608320114545, 0.0017146199789218066 ], "confidence_level": 0.95, "mean": -0.00022867540751119256, "replicates": 8, "sample_sd": 0.0025990185162106686, "standard_error": 0.0009188918086209813 }, "log_loss": { "bootstrap_interval": [ -0.026892676501988115, -0.004500623040807845 ], "confidence_level": 0.95, "mean": -0.016497768322830387, "replicates": 8, "sample_sd": 0.017563199923795264, "standard_error": 0.006209528882725342 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0003971458734490776, 0.0030833804604727243 ], "confidence_level": 0.95, "mean": 0.0014848193594592662, "replicates": 8, "sample_sd": 0.0021606130060831397, "standard_error": 0.0007638920540606197 }, "log_loss": { "bootstrap_interval": [ 0.005058957482468852, 0.023251907749115456 ], "confidence_level": 0.95, "mean": 0.013830889892755731, "replicates": 8, "sample_sd": 0.014375179564107426, "standard_error": 0.005082393475277319 } } }, "discount": { "bootstrap_interval": [ 0.4564726998464446, 0.7749265305082796 ], "confidence_level": 0.95, "mean": 0.6134672751444555, "replicates": 8, "sample_sd": 0.24384222666208194, "standard_error": 0.08621124600619265 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.2797219583240259, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.07071640253113434, 0.07411112418060425 ], "confidence_level": 0.95, "mean": 0.07244672941212359, "replicates": 8, "sample_sd": 0.002653629303197717, "standard_error": 0.0009381996375232193 }, "log_loss": { "bootstrap_interval": [ 0.2716331132218428, 0.2810526133762792 ], "confidence_level": 0.95, "mean": 0.27648453452301225, "replicates": 8, "sample_sd": 0.007397094573614432, "standard_error": 0.002615267867040489 } }, "independent": { "brier": { "bootstrap_interval": [ 0.03858314799601826, 0.04286232653815273 ], "confidence_level": 0.95, "mean": 0.040729619620513344, "replicates": 8, "sample_sd": 0.0032826423507549864, "standard_error": 0.0011605893332145 }, "log_loss": { "bootstrap_interval": [ 0.15474807985833813, 0.1753159525103946 ], "confidence_level": 0.95, "mean": 0.16519204091070588, "replicates": 8, "sample_sd": 0.01617660481975416, "standard_error": 0.005719293482311577 } }, "individual": { "brier": { "bootstrap_interval": [ 0.11072065688134883, 0.11674986911993418 ], "confidence_level": 0.95, "mean": 0.1136443487378146, "replicates": 8, "sample_sd": 0.004602628198623606, "standard_error": 0.0016272748052635878 }, "log_loss": { "bootstrap_interval": [ 0.3507613479634174, 0.3676153333989287 ], "confidence_level": 0.95, "mean": 0.3590003481495725, "replicates": 8, "sample_sd": 0.0128353060091697, "standard_error": 0.004537965958844168 } }, "learned": { "brier": { "bootstrap_interval": [ 0.03907847739897911, 0.04185867553267251 ], "confidence_level": 0.95, "mean": 0.040500944213002155, "replicates": 8, "sample_sd": 0.0021504827276225182, "standard_error": 0.0007603104597632129 }, "log_loss": { "bootstrap_interval": [ 0.1402272184315008, 0.1567665234935008 ], "confidence_level": 0.95, "mean": 0.1486942725878755, "replicates": 8, "sample_sd": 0.012730802588660715, "standard_error": 0.004501018420194622 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.037534901287955696, 0.040518866650453816 ], "confidence_level": 0.95, "mean": 0.03901612485354289, "replicates": 8, "sample_sd": 0.0023142007399986056, "standard_error": 0.0008181935181399701 }, "log_loss": { "bootstrap_interval": [ 0.12996827813302017, 0.13964486851092403 ], "confidence_level": 0.95, "mean": 0.13486338269511974, "replicates": 8, "sample_sd": 0.007736843709248356, "standard_error": 0.002735387325894997 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.5 ], "oracle_method": "oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.5-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.008053400807878374, -0.0029138578065229267 ], "confidence_level": 0.95, "mean": -0.005905607385839954, "replicates": 8, "sample_sd": 0.004040581999211401, "standard_error": 0.0014285614657913394 }, "log_loss": { "bootstrap_interval": [ -0.02412830512774268, 0.016122385964738792 ], "confidence_level": 0.95, "mean": -0.009310090356603129, "replicates": 8, "sample_sd": 0.03514203482496141, "standard_error": 0.012424585564712008 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.02832754682786002, -0.017015917712313273 ], "confidence_level": 0.95, "mean": -0.023299276558118094, "replicates": 8, "sample_sd": 0.0088485404461562, "standard_error": 0.003128431476540244 }, "log_loss": { "bootstrap_interval": [ -0.21172421431966323, -0.1488175285446101 ], "confidence_level": 0.95, "mean": -0.1847801156488162, "replicates": 8, "sample_sd": 0.050664293798734335, "standard_error": 0.017912532854556296 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010060610465348924, 0.007273279440829793 ], "confidence_level": 0.95, "mean": 0.0036202448748973645, "replicates": 8, "sample_sd": 0.005024665045862169, "standard_error": 0.0017764873635600772 }, "log_loss": { "bootstrap_interval": [ 0.0037862087037192488, 0.04693703646772696 ], "confidence_level": 0.95, "mean": 0.019999719957821745, "replicates": 8, "sample_sd": 0.037247223475972174, "standard_error": 0.013168882150115345 } } }, "discount": { "bootstrap_interval": [ 0.2765085376657756, 0.5220070919010642 ], "confidence_level": 0.95, "mean": 0.379359783210884, "replicates": 8, "sample_sd": 0.19454090007963204, "standard_error": 0.06878059483222118 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.8251854032018564, "discount_min": 0.22508168827671615, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17884414822810032, 0.1854012235593562 ], "confidence_level": 0.95, "mean": 0.1820843610446205, "replicates": 8, "sample_sd": 0.005142600321953683, "standard_error": 0.0018181837802927858 }, "log_loss": { "bootstrap_interval": [ 0.5386764853735766, 0.5529774660078611 ], "confidence_level": 0.95, "mean": 0.5456882287093854, "replicates": 8, "sample_sd": 0.011141361764504058, "standard_error": 0.003939066227666669 } }, "independent": { "brier": { "bootstrap_interval": [ 0.19240278892966559, 0.2063141778014865 ], "confidence_level": 0.95, "mean": 0.19947803021689864, "replicates": 8, "sample_sd": 0.010857725506550255, "standard_error": 0.0038387856669719134 }, "log_loss": { "bootstrap_interval": [ 0.6990159396953428, 0.742633104606566 ], "confidence_level": 0.95, "mean": 0.7211582540015985, "replicates": 8, "sample_sd": 0.033737660320993644, "standard_error": 0.011928064197171458 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19535655318512105, 0.20361257961864107 ], "confidence_level": 0.95, "mean": 0.19920056907551684, "replicates": 8, "sample_sd": 0.00638457994517526, "standard_error": 0.002257289887130531 }, "log_loss": { "bootstrap_interval": [ 0.5735803161103621, 0.5920840764381631 ], "confidence_level": 0.95, "mean": 0.5821224059622203, "replicates": 8, "sample_sd": 0.01413800323726898, "standard_error": 0.0049985389807551284 } }, "learned": { "brier": { "bootstrap_interval": [ 0.17204334281443756, 0.18031152022968833 ], "confidence_level": 0.95, "mean": 0.17617875365878055, "replicates": 8, "sample_sd": 0.00643031076306062, "standard_error": 0.002273458172848503 }, "log_loss": { "bootstrap_interval": [ 0.5181639535394427, 0.5620902207083472 ], "confidence_level": 0.95, "mean": 0.5363781383527823, "replicates": 8, "sample_sd": 0.035562023288852065, "standard_error": 0.012573073910130612 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.16793259944672434, 0.17701326777631554 ], "confidence_level": 0.95, "mean": 0.17255850878388318, "replicates": 8, "sample_sd": 0.0070207650508500655, "standard_error": 0.0024822152882867984 }, "log_loss": { "bootstrap_interval": [ 0.5058580033661911, 0.5267728938417486 ], "confidence_level": 0.95, "mean": 0.5163784183949606, "replicates": 8, "sample_sd": 0.01629410483683, "standard_error": 0.005760836011743508 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.5-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.014546752301683955, -0.009810324759415541 ], "confidence_level": 0.95, "mean": -0.012415119252809283, "replicates": 8, "sample_sd": 0.0037163036071881273, "standard_error": 0.001313911740795376 }, "log_loss": { "bootstrap_interval": [ -0.05852252226971302, -0.007086726212425835 ], "confidence_level": 0.95, "mean": -0.03791152162749689, "replicates": 8, "sample_sd": 0.04187559351167556, "standard_error": 0.014805258069158589 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0077975385580746435, -0.00388153127505853 ], "confidence_level": 0.95, "mean": -0.005954727965058413, "replicates": 8, "sample_sd": 0.0030616136662297696, "standard_error": 0.0010824438923822386 }, "log_loss": { "bootstrap_interval": [ -0.13229814255726186, -0.07519785169403685 ], "confidence_level": 0.95, "mean": -0.10925034178453462, "replicates": 8, "sample_sd": 0.046764611784756834, "standard_error": 0.016533787056278944 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0006762524357423784, 0.004840488456333761 ], "confidence_level": 0.95, "mean": 0.0024723697351221163, "replicates": 8, "sample_sd": 0.0033215262162180023, "standard_error": 0.0011743368556883218 }, "log_loss": { "bootstrap_interval": [ 0.006677695923985608, 0.0588955901452995 ], "confidence_level": 0.95, "mean": 0.02735220675727104, "replicates": 8, "sample_sd": 0.04302420956989437, "standard_error": 0.01521135517103173 } } }, "discount": { "bootstrap_interval": [ 0.3019893421545304, 0.6327980456618062 ], "confidence_level": 0.95, "mean": 0.4481452465267831, "replicates": 8, "sample_sd": 0.2584784559614399, "standard_error": 0.09138593450048127 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.21761880171640813, "discount_one_count": 1, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.08102123122195079, 0.08585403615429124 ], "confidence_level": 0.95, "mean": 0.08352525845278212, "replicates": 8, "sample_sd": 0.003715903740942498, "standard_error": 0.00131377036672845 }, "log_loss": { "bootstrap_interval": [ 0.28603008878791647, 0.298353449280364 ], "confidence_level": 0.95, "mean": 0.2924710602049817, "replicates": 8, "sample_sd": 0.009689308384855063, "standard_error": 0.0034256878319693447 } }, "independent": { "brier": { "bootstrap_interval": [ 0.07446107902695373, 0.07978374292419305 ], "confidence_level": 0.95, "mean": 0.07706486716503125, "replicates": 8, "sample_sd": 0.004149803399955308, "standard_error": 0.0014671770623496945 }, "log_loss": { "bootstrap_interval": [ 0.35290445829577555, 0.3759861465844292 ], "confidence_level": 0.95, "mean": 0.3638098803620194, "replicates": 8, "sample_sd": 0.01800577995446044, "standard_error": 0.006366004553175891 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10880930129601654, 0.11580885497434946 ], "confidence_level": 0.95, "mean": 0.11230572728599801, "replicates": 8, "sample_sd": 0.005498152457137541, "standard_error": 0.0019438904432197166 }, "log_loss": { "bootstrap_interval": [ 0.34654577282114546, 0.3654873983905853 ], "confidence_level": 0.95, "mean": 0.3558041579520046, "replicates": 8, "sample_sd": 0.014597153738659536, "standard_error": 0.005160873197314361 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0681766350674668, 0.07374236097641243 ], "confidence_level": 0.95, "mean": 0.07111013919997283, "replicates": 8, "sample_sd": 0.004310658071094493, "standard_error": 0.0015240477767237194 }, "log_loss": { "bootstrap_interval": [ 0.23307838291480704, 0.28390790932591004 ], "confidence_level": 0.95, "mean": 0.2545595385774848, "replicates": 8, "sample_sd": 0.04066664485514261, "standard_error": 0.014377830172588181 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.06635643985198264, 0.0709037068262689 ], "confidence_level": 0.95, "mean": 0.06863776946485073, "replicates": 8, "sample_sd": 0.0034656583543369837, "standard_error": 0.001225295261813746 }, "log_loss": { "bootstrap_interval": [ 0.22105433051914788, 0.23372779948142602 ], "confidence_level": 0.95, "mean": 0.22720733182021374, "replicates": 8, "sample_sd": 0.009690428134484872, "standard_error": 0.003426083723247579 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333 ], "oracle_method": "oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.75-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0006059439206722861, 0.0030895142765221535 ], "confidence_level": 0.95, "mean": 0.0010863224175942376, "replicates": 8, "sample_sd": 0.0028414186692043383, "standard_error": 0.0010045932045922214 }, "log_loss": { "bootstrap_interval": [ -0.002131363913714509, 0.017583181805459355 ], "confidence_level": 0.95, "mean": 0.005923212492203844, "replicates": 8, "sample_sd": 0.015792347183331744, "standard_error": 0.005583437892093074 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.043775140875400236, -0.03049660938100951 ], "confidence_level": 0.95, "mean": -0.037807407225233625, "replicates": 8, "sample_sd": 0.010430608358589942, "standard_error": 0.0036877769511300157 }, "log_loss": { "bootstrap_interval": [ -0.3668245339336216, -0.30389536190678207 ], "confidence_level": 0.95, "mean": -0.33746340462921554, "replicates": 8, "sample_sd": 0.04925351910455122, "standard_error": 0.017413748678064667 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010727887756218748, 0.006043664059396157 ], "confidence_level": 0.95, "mean": 0.0031962279551120756, "replicates": 8, "sample_sd": 0.003705208060522223, "standard_error": 0.0013099888726511597 }, "log_loss": { "bootstrap_interval": [ 0.0037274141622340843, 0.024977309998627323 ], "confidence_level": 0.95, "mean": 0.012721070909366597, "replicates": 8, "sample_sd": 0.017273811582239625, "standard_error": 0.0061072146533701814 } } }, "discount": { "bootstrap_interval": [ 0.20903731466393394, 0.3518678489023662 ], "confidence_level": 0.95, "mean": 0.2737987400414909, "replicates": 8, "sample_sd": 0.10992448885633917, "standard_error": 0.03886417574439125 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.5032697074988657, "discount_min": 0.15932695199261648, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18595956867464103, 0.19408213322244902 ], "confidence_level": 0.95, "mean": 0.19004695461492982, "replicates": 8, "sample_sd": 0.006249835506461598, "standard_error": 0.002209650533959728 }, "log_loss": { "bootstrap_interval": [ 0.5528237110573804, 0.5700768799336714 ], "confidence_level": 0.95, "mean": 0.5614586015803581, "replicates": 8, "sample_sd": 0.013499597680769101, "standard_error": 0.004772828531681011 } }, "independent": { "brier": { "bootstrap_interval": [ 0.21959660026658365, 0.23718526827377592 ], "confidence_level": 0.95, "mean": 0.2289406842577577, "replicates": 8, "sample_sd": 0.013552683747957504, "standard_error": 0.004791597290728733 }, "log_loss": { "bootstrap_interval": [ 0.8698553783831977, 0.9353245622815656 ], "confidence_level": 0.95, "mean": 0.9048452187017775, "replicates": 8, "sample_sd": 0.05058844265590705, "standard_error": 0.017885715425829334 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1942246981298946, 0.20339267184464438 ], "confidence_level": 0.95, "mean": 0.19869529628463112, "replicates": 8, "sample_sd": 0.007071188646351022, "standard_error": 0.0025000427214420654 }, "log_loss": { "bootstrap_interval": [ 0.5712888413695788, 0.5912025810789452 ], "confidence_level": 0.95, "mean": 0.58080379894007, "replicates": 8, "sample_sd": 0.015418508102089555, "standard_error": 0.005451265817383624 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18730122800625998, 0.19500524299868213 ], "confidence_level": 0.95, "mean": 0.19113327703252408, "replicates": 8, "sample_sd": 0.005979839313410512, "standard_error": 0.0021141924644592406 }, "log_loss": { "bootstrap_interval": [ 0.5557827096212731, 0.5805550954365208 ], "confidence_level": 0.95, "mean": 0.5673818140725619, "replicates": 8, "sample_sd": 0.0188907908016001, "standard_error": 0.006678903138893943 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.18299168327480544, 0.19250487359130705 ], "confidence_level": 0.95, "mean": 0.187937049077412, "replicates": 8, "sample_sd": 0.007363685423680721, "standard_error": 0.0026034559488045863 }, "log_loss": { "bootstrap_interval": [ 0.5437525529883308, 0.5653989955305105 ], "confidence_level": 0.95, "mean": 0.5546607431631954, "replicates": 8, "sample_sd": 0.01642658627866368, "standard_error": 0.00580767527469449 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25 ], "oracle_method": "oracle", "population_optimal_scalar": 0.25, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho0.75-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.003557653273229598, 0.00226635007908336 ], "confidence_level": 0.95, "mean": -0.0012367680341714756, "replicates": 8, "sample_sd": 0.004742410051839094, "standard_error": 0.0016766951534113346 }, "log_loss": { "bootstrap_interval": [ -0.013012463142010176, 0.05665363901293663 ], "confidence_level": 0.95, "mean": 0.011922753590494453, "replicates": 8, "sample_sd": 0.061147289566922086, "standard_error": 0.021618831551974017 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.015835352373351454, -0.00929764878381633 ], "confidence_level": 0.95, "mean": -0.012837838035710373, "replicates": 8, "sample_sd": 0.005066702670248922, "standard_error": 0.0017913499081945 }, "log_loss": { "bootstrap_interval": [ -0.29557428921242396, -0.20811809402012424 ], "confidence_level": 0.95, "mean": -0.26115998381703853, "replicates": 8, "sample_sd": 0.0710018972906258, "standard_error": 0.025102961525656126 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0007986567520322622, 0.0063190040600583485 ], "confidence_level": 0.95, "mean": 0.002891896410531657, "replicates": 8, "sample_sd": 0.004641184950274204, "standard_error": 0.0016409066755399195 }, "log_loss": { "bootstrap_interval": [ 0.005505085510794576, 0.07581965886819969 ], "confidence_level": 0.95, "mean": 0.03058121024904488, "replicates": 8, "sample_sd": 0.06212048267262243, "standard_error": 0.02196290727419637 } } }, "discount": { "bootstrap_interval": [ 0.22524063524249183, 0.48286592946958024 ], "confidence_level": 0.95, "mean": 0.33049183634949164, "replicates": 8, "sample_sd": 0.20535401819943438, "standard_error": 0.07260360940636287 }, "discount_below_exchangeable_range_count": 2, "discount_max": 0.8059711415884856, "discount_min": 0.1754212878454448, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.09240592483875745, 0.09897948508399015 ], "confidence_level": 0.95, "mean": 0.09588171977760739, "replicates": 8, "sample_sd": 0.005205056964727235, "standard_error": 0.0018402655381104481 }, "log_loss": { "bootstrap_interval": [ 0.3067361720951051, 0.32293557902388254 ], "confidence_level": 0.95, "mean": 0.3153580042374583, "replicates": 8, "sample_sd": 0.012561019987242392, "standard_error": 0.004440991205799428 } }, "independent": { "brier": { "bootstrap_interval": [ 0.10124010978662629, 0.11268557532465645 ], "confidence_level": 0.95, "mean": 0.10748278977914628, "replicates": 8, "sample_sd": 0.008826515182333875, "standard_error": 0.0031206443698371493 }, "log_loss": { "bootstrap_interval": [ 0.564376845842476, 0.6095789694118503 ], "confidence_level": 0.95, "mean": 0.5884407416449913, "replicates": 8, "sample_sd": 0.034809926490741744, "standard_error": 0.012307167537104362 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10701533168248309, 0.11524651413904463 ], "confidence_level": 0.95, "mean": 0.11121122109872107, "replicates": 8, "sample_sd": 0.006408123641965544, "standard_error": 0.002265613840957836 }, "log_loss": { "bootstrap_interval": [ 0.3427449764876107, 0.3626363876655749 ], "confidence_level": 0.95, "mean": 0.3526925037658831, "replicates": 8, "sample_sd": 0.01593143496684411, "standard_error": 0.005632612849543975 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09000790616773395, 0.09865181882037566 ], "confidence_level": 0.95, "mean": 0.0946449517434359, "replicates": 8, "sample_sd": 0.0066680828464978055, "standard_error": 0.002357523299136147 }, "log_loss": { "bootstrap_interval": [ 0.2985668053692478, 0.3709621094003463 ], "confidence_level": 0.95, "mean": 0.32728075782795274, "replicates": 8, "sample_sd": 0.059678054353536375, "standard_error": 0.021099378460702468 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.08768361607293002, 0.09515349187606746 ], "confidence_level": 0.95, "mean": 0.09175305533290425, "replicates": 8, "sample_sd": 0.005779009569977078, "standard_error": 0.0020431884277363724 }, "log_loss": { "bootstrap_interval": [ 0.28719102351541603, 0.3052389074198884 ], "confidence_level": 0.95, "mean": 0.2966995475789079, "replicates": 8, "sample_sd": 0.01412322081528342, "standard_error": 0.0049933126053409525 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25 ], "oracle_method": "oracle", "population_optimal_scalar": 0.25, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho1-a0.5": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0010671116504228709, 0.005007400526200221 ], "confidence_level": 0.95, "mean": 0.0026696714486234216, "replicates": 8, "sample_sd": 0.0032450327229714957, "standard_error": 0.0011472923217926958 }, "log_loss": { "bootstrap_interval": [ 0.003582676303796828, 0.01410642860741078 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.05562465256124256, -0.042370290442518554 ], "confidence_level": 0.95, "mean": -0.04970179247144352, "replicates": 8, "sample_sd": 0.010414977889257883, "standard_error": 0.003682250745701102 }, "log_loss": { "bootstrap_interval": [ -0.5232450257045034, -0.44528750697573305 ], "confidence_level": 0.95, "mean": -0.48802410023721055, "replicates": 8, "sample_sd": 0.06131875479633236, "standard_error": 0.02167945366520087 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0010671116504228709, 0.005007400526200221 ], "confidence_level": 0.95, "mean": 0.0026696714486234216, "replicates": 8, "sample_sd": 0.0032450327229714957, "standard_error": 0.0011472923217926958 }, "log_loss": { "bootstrap_interval": [ 0.003582676303796828, 0.01410642860741078 ], "confidence_level": 0.95, "mean": 0.008214525923310262, "replicates": 8, "sample_sd": 0.008371373593551609, "standard_error": 0.0029597275179231693 } } }, "discount": { "bootstrap_interval": [ 0.1751474134985366, 0.2555920324214768 ], "confidence_level": 0.95, "mean": 0.21538417171470442, "replicates": 8, "sample_sd": 0.0630682138550478, "standard_error": 0.022297980847113832 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.30369510189540516, "discount_min": 0.12298862031679081, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19306915389124474, 0.20256519485982485 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5683179919087981, 0.5892435495742444 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2405304225830744, 0.25901189059193497 ], "confidence_level": 0.95, "mean": 0.2504550277049001, "replicates": 8, "sample_sd": 0.014369694276697136, "standard_error": 0.0050804541333150325 }, "log_loss": { "bootstrap_interval": [ 1.0277447815322878, 1.1158941285077741 ], "confidence_level": 0.95, "mean": 1.075509146240288, "replicates": 8, "sample_sd": 0.06902874507106078, "standard_error": 0.02440534686827227 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19302814282628636, 0.20250407403395518 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5684267353075054, 0.5892310518455174 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19668297752326774, 0.20483026302537632 ], "confidence_level": 0.95, "mean": 0.20075323523345656, "replicates": 8, "sample_sd": 0.006395098492831272, "standard_error": 0.002261008755318431 }, "log_loss": { "bootstrap_interval": [ 0.5775542063252488, 0.5972672189131694 ], "confidence_level": 0.95, "mean": 0.5874850460030775, "replicates": 8, "sample_sd": 0.015094497522713133, "standard_error": 0.005336710778456999 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.19299445563373221, 0.20267735788729002 ], "confidence_level": 0.95, "mean": 0.19808356378483313, "replicates": 8, "sample_sd": 0.007419249306777652, "standard_error": 0.0026231007480680348 }, "log_loss": { "bootstrap_interval": [ 0.5681380023434431, 0.5893666054135683 ], "confidence_level": 0.95, "mean": 0.5792705200797672, "replicates": 8, "sample_sd": 0.01624206981582175, "standard_error": 0.005742438853636448 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2 ], "oracle_method": "oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 }, "exchangeable-n5-rho1-a1": { "condition": { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.0008248752927088564, 0.0034426057462891204 ], "confidence_level": 0.95, "mean": 0.002033601459410844, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.004665896778604892, 0.019934173331520956 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922939, "standard_error": 0.004312314006328675 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.02561317987642449, -0.017906332080097405 ], "confidence_level": 0.95, "mean": -0.021886496091759376, "replicates": 8, "sample_sd": 0.006005920688408357, "standard_error": 0.0021234136230210633 }, "log_loss": { "bootstrap_interval": [ -0.5020681703224901, -0.4267909978951217 ], "confidence_level": 0.95, "mean": -0.4668632807424009, "replicates": 8, "sample_sd": 0.058421274922100086, "standard_error": 0.02065503983149028 } }, "learned_minus_oracle": { "brier": { "bootstrap_interval": [ 0.0008248752927088564, 0.0034426057462891204 ], "confidence_level": 0.95, "mean": 0.002033601459410844, "replicates": 8, "sample_sd": 0.0020350714159422813, "standard_error": 0.000719506399205848 }, "log_loss": { "bootstrap_interval": [ 0.004665896778604892, 0.019934173331520956 ], "confidence_level": 0.95, "mean": 0.011483691699256927, "replicates": 8, "sample_sd": 0.012197065905922939, "standard_error": 0.004312314006328675 } } }, "discount": { "bootstrap_interval": [ 0.1811597882670447, 0.27508573667666486 ], "confidence_level": 0.95, "mean": 0.22596221287671348, "replicates": 8, "sample_sd": 0.07328244920813562, "standard_error": 0.025909258388515717 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.36242821156912597, "discount_min": 0.14102348372108586, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1047822158964306, 0.1142401214711498 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.33753668195578407, 0.36020506541988634 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "independent": { "brier": { "bootstrap_interval": [ 0.12554119017103918, 0.1412540474422706 ], "confidence_level": 0.95, "mean": 0.13383173997400769, "replicates": 8, "sample_sd": 0.012396802278726719, "standard_error": 0.004382931478158254 }, "log_loss": { "bootstrap_interval": [ 0.7791876994752207, 0.8688663998286226 ], "confidence_level": 0.95, "mean": 0.8277343599198186, "replicates": 8, "sample_sd": 0.06935354204536175, "standard_error": 0.024520179939790814 } }, "individual": { "brier": { "bootstrap_interval": [ 0.10469301968980727, 0.11419638152353973 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.3369986976619657, 0.360308104775459 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "learned": { "brier": { "bootstrap_interval": [ 0.10727381944771078, 0.11593325083791631 ], "confidence_level": 0.95, "mean": 0.1119452438822483, "replicates": 8, "sample_sd": 0.006750192655061165, "standard_error": 0.002386553500354688 }, "log_loss": { "bootstrap_interval": [ 0.3488117497793217, 0.37191276883584956 ], "confidence_level": 0.95, "mean": 0.3608710791774176, "replicates": 8, "sample_sd": 0.01801647685394999, "standard_error": 0.0063697864782592565 } }, "oracle": { "brier": { "bootstrap_interval": [ 0.10475878146918542, 0.11420147875333718 ], "confidence_level": 0.95, "mean": 0.10991164242283746, "replicates": 8, "sample_sd": 0.0073925897991049235, "standard_error": 0.002613675188738794 }, "log_loss": { "bootstrap_interval": [ 0.33687978914709854, 0.3601965142964396 ], "confidence_level": 0.95, "mean": 0.3493873874781607, "replicates": 8, "sample_sd": 0.01804673757447612, "standard_error": 0.006380485258603065 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 8, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2 ], "oracle_method": "oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 1000, "train_count_per_replicate": 50 } }, "config": { "conditions": [ { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 2, "name": "exchangeable-n2-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 5, "name": "exchangeable-n5-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 10, "name": "exchangeable-n10-rho1-a1", "overlap": 1, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a0.5", "overlap": 0, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0-a1", "overlap": 0, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a0.5", "overlap": 0.1, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.1-a1", "overlap": 0.1, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a0.5", "overlap": 0.25, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.25-a1", "overlap": 0.25, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.5-a1", "overlap": 0.5, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a0.5", "overlap": 0.75, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho0.75-a1", "overlap": 0.75, "signal_strength": 1.0 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a0.5", "overlap": 1, "signal_strength": 0.5 }, { "n_forecasters": 20, "name": "exchangeable-n20-rho1-a1", "overlap": 1, "signal_strength": 1.0 } ], "phase": "exploratory-stress-exchangeable-m50-v1", "replicates": 8, "schema_version": 1, "seed": 84173000, "test_count": 1000, "train_count": 50 }, "config_sha256": "e2666ff875a37346b3bb9cf869f1bfc6bb6679c593493b340844e501e0f298ae", "expected_runs": 384, "input_path": "results/stress-v1/stress-exchangeable-m50-v1.json", "input_sha256": "d7d9d387fd639a190bc7099352111ec93fc6ae492770dfa9b2776b21133e80c2", "ledger_receipt": { "bytes": 122523771, "count": 403200, "payload_sha256": "5c63d733ec19c51c9be1ceabc500205045cdba714a63fef25723a8041eaa0cc3", "schema_version": 1, "sha256": "cdf9c404be5988f876b2ade87b07a24ffc695790a4986be7937f8d6f578dfcf0" }, "manifest_sha256": null, "observed_runs": 384, "phase": "exploratory-stress-exchangeable-m50-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" }, "exploratory-stress-groups-m1000-v1": { "analysis_seed": 45910105237656156229960997419600076665555110031689186237100746007021009975958, "cells": { "groups-4-3-2-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12167726681969517, -0.11949840653278739 ], "confidence_level": 0.95, "mean": -0.12064721933313702, "replicates": 12, "sample_sd": 0.001984274791145078, "standard_error": 0.0005728107924068996 }, "log_loss": { "bootstrap_interval": [ -0.3751767640171788, -0.36770934600075916 ], "confidence_level": 0.95, "mean": -0.3716041505299348, "replicates": 12, "sample_sd": 0.006819939578562517, "standard_error": 0.0019687469757700264 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 1.3481329537088356e-05, 0.00012935159579212366 ], "confidence_level": 0.95, "mean": 6.599135730207238e-05, "replicates": 12, "sample_sd": 0.00010642515210320216, "standard_error": 3.0722295107665316e-05 }, "log_loss": { "bootstrap_interval": [ 0.00022490604886021606, 0.0014163897624520385 ], "confidence_level": 0.95, "mean": 0.0007536849467200841, "replicates": 12, "sample_sd": 0.0011100584009282341, "standard_error": 0.00032044625829606076 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 1.3481329537084336e-05, 0.00012935159579212363 ], "confidence_level": 0.95, "mean": 6.599135730207123e-05, "replicates": 12, "sample_sd": 0.00010642515210320401, "standard_error": 3.072229510766585e-05 }, "log_loss": { "bootstrap_interval": [ 0.00022490604886021837, 0.001416389762452043 ], "confidence_level": 0.95, "mean": 0.0007536849467200865, "replicates": 12, "sample_sd": 0.0011100584009282324, "standard_error": 0.00032044625829606027 } } }, "discount": { "bootstrap_interval": [ 0.9144827848051914, 0.9763045578266785 ], "confidence_level": 0.95, "mean": 0.9466173145612724, "replicates": 12, "sample_sd": 0.05707188629628911, "standard_error": 0.016475234458161115 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8477297425356989, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1616075138344032, 0.16291059475779995 ], "confidence_level": 0.95, "mean": 0.16229312162995393, "replicates": 12, "sample_sd": 0.001206242253059362, "standard_error": 0.0003482121447558617 }, "log_loss": { "bootstrap_interval": [ 0.5114718401964197, 0.5141728926395892 ], "confidence_level": 0.95, "mean": 0.5129265246198856, "replicates": 12, "sample_sd": 0.002501743789771024, "standard_error": 0.0007221912252338877 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.04010441114401966, 0.043161527113090334 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712502, "standard_error": 0.0008155893598404593 }, "log_loss": { "bootstrap_interval": [ 0.13588206451551046, 0.14542061595971195 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.04008463108726121, 0.04313257811318588 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712503, "standard_error": 0.0008155893598404595 }, "log_loss": { "bootstrap_interval": [ 0.13587120242884942, 0.14542139119155345 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04007082213922754, 0.04312237134622829 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712502, "standard_error": 0.0008155893598404593 }, "log_loss": { "bootstrap_interval": [ 0.1358679219436949, 0.14538134103897882 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19834661289380084, 0.2030401281948187 ], "confidence_level": 0.95, "mean": 0.20076021787067505, "replicates": 12, "sample_sd": 0.004305894758847835, "standard_error": 0.0012430047490614982 }, "log_loss": { "bootstrap_interval": [ 0.5800707606527392, 0.5907823320423482 ], "confidence_level": 0.95, "mean": 0.5857046061562484, "replicates": 12, "sample_sd": 0.009804608943974726, "standard_error": 0.0028303468065514104 } }, "learned": { "brier": { "bootstrap_interval": [ 0.040137130991521774, 0.0432070355116022 ], "confidence_level": 0.95, "mean": 0.04164590229681692, "replicates": 12, "sample_sd": 0.002810023435715211, "standard_error": 0.000811183893519667 }, "log_loss": { "bootstrap_interval": [ 0.13672955352672028, 0.1462014060712563 ], "confidence_level": 0.95, "mean": 0.14132237408995083, "replicates": 12, "sample_sd": 0.008621266070041886, "standard_error": 0.002488745143147035 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0, 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-4-3-2-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07392754306398579, -0.07080582509980367 ], "confidence_level": 0.95, "mean": -0.07237296011321549, "replicates": 12, "sample_sd": 0.002900184527737291, "standard_error": 0.0008372111588943564 }, "log_loss": { "bootstrap_interval": [ -0.2200115414601204, -0.21115661068183156 ], "confidence_level": 0.95, "mean": -0.21551840912636527, "replicates": 12, "sample_sd": 0.00808919466940986, "standard_error": 0.0023351493599555345 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.006830861632820152, 0.007986612791256143 ], "confidence_level": 0.95, "mean": 0.007423206988314537, "replicates": 12, "sample_sd": 0.0010886720891877366, "standard_error": 0.00031427256187588603 }, "log_loss": { "bootstrap_interval": [ 0.019908057030343086, 0.023259796972783048 ], "confidence_level": 0.95, "mean": 0.021664609078111432, "replicates": 12, "sample_sd": 0.0031514492874170114, "standard_error": 0.000909745047213833 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.007325715516418209, -0.005874311741403361 ], "confidence_level": 0.95, "mean": -0.006622697291176882, "replicates": 12, "sample_sd": 0.001322642212732748, "standard_error": 0.0003818139187814072 }, "log_loss": { "bootstrap_interval": [ -0.07269511767552894, -0.0638574675109657 ], "confidence_level": 0.95, "mean": -0.06847382355103425, "replicates": 12, "sample_sd": 0.008137141194663725, "standard_error": 0.0023489903295865473 } } }, "discount": { "bootstrap_interval": [ 0.4877895215354775, 0.5125232913053321 ], "confidence_level": 0.95, "mean": 0.5002011960000612, "replicates": 12, "sample_sd": 0.022722982629587028, "standard_error": 0.006559560068991631 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.5287168740804544, "discount_min": 0.467619746840512, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16568434660838532, 0.16730992255657073 ], "confidence_level": 0.95, "mean": 0.16646941849161875, "replicates": 12, "sample_sd": 0.0015186463396758287, "standard_error": 0.0004383954365078398 }, "log_loss": { "bootstrap_interval": [ 0.5176217211358008, 0.5210423825109036 ], "confidence_level": 0.95, "mean": 0.5192774423173853, "replicates": 12, "sample_sd": 0.0031784074172865107, "standard_error": 0.0009175271889823351 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.11963854420288236, 0.12243331209064046 ], "confidence_level": 0.95, "mean": 0.120977446591782, "replicates": 12, "sample_sd": 0.0025658502619266537, "standard_error": 0.0007406971697118127 }, "log_loss": { "bootstrap_interval": [ 0.4005927721713805, 0.4067637445049959 ], "confidence_level": 0.95, "mean": 0.40357437519769546, "replicates": 12, "sample_sd": 0.005688183003206596, "standard_error": 0.0016420369940505911 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.08433878913999317, 0.08901597327004948 ], "confidence_level": 0.95, "mean": 0.08667325139008872, "replicates": 12, "sample_sd": 0.004270821449727733, "standard_error": 0.0012328799568305671 }, "log_loss": { "bootstrap_interval": [ 0.27647058321250434, 0.28770955610839466 ], "confidence_level": 0.95, "mean": 0.2820944241129085, "replicates": 12, "sample_sd": 0.010429736790237185, "standard_error": 0.003010805671710191 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0978939208114681, 0.10373461404455789 ], "confidence_level": 0.95, "mean": 0.10071915566958015, "replicates": 12, "sample_sd": 0.0053936590793036195, "standard_error": 0.001557015260676507 }, "log_loss": { "bootstrap_interval": [ 0.36188568477222166, 0.3821198139764414 ], "confidence_level": 0.95, "mean": 0.3722328567420542, "replicates": 12, "sample_sd": 0.018958893856728925, "standard_error": 0.005472961235859994 } }, "individual": { "brier": { "bootstrap_interval": [ 0.198603947811913, 0.20231432901906207 ], "confidence_level": 0.95, "mean": 0.20051126014712403, "replicates": 12, "sample_sd": 0.0034439886434354883, "standard_error": 0.0009941938851867466 }, "log_loss": { "bootstrap_interval": [ 0.5813077061546513, 0.5897059847607522 ], "confidence_level": 0.95, "mean": 0.5857271248029461, "replicates": 12, "sample_sd": 0.007748180206795777, "standard_error": 0.0022367069640616363 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09179271980533835, 0.09642475831307114 ], "confidence_level": 0.95, "mean": 0.09409645837840326, "replicates": 12, "sample_sd": 0.004246792791201772, "standard_error": 0.0012259434805964526 }, "log_loss": { "bootstrap_interval": [ 0.2976560380904072, 0.3097272408401385 ], "confidence_level": 0.95, "mean": 0.30375903319101993, "replicates": 12, "sample_sd": 0.011062085719593233, "standard_error": 0.0031933490840029342 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.4, 0.5, 0.6666666666666666, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-4-3-2-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04992337240093006, -0.04734667421834113 ], "confidence_level": 0.95, "mean": -0.04879181494436279, "replicates": 12, "sample_sd": 0.0024388083128903675, "standard_error": 0.0007040233179745754 }, "log_loss": { "bootstrap_interval": [ -0.14633008565654618, -0.13949150475363375 ], "confidence_level": 0.95, "mean": -0.14318725425974083, "replicates": 12, "sample_sd": 0.006466119726270002, "standard_error": 0.0018666079822871676 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.012686421453336167, 0.014318668295646155 ], "confidence_level": 0.95, "mean": 0.013480653392444239, "replicates": 12, "sample_sd": 0.0015225656968707551, "standard_error": 0.0004395268574736103 }, "log_loss": { "bootstrap_interval": [ 0.03591260444264261, 0.04033628364641125 ], "confidence_level": 0.95, "mean": 0.038069657344810236, "replicates": 12, "sample_sd": 0.004137394730491352, "standard_error": 0.0011943629806964607 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.01682882783476605, -0.014397924777210233 ], "confidence_level": 0.95, "mean": -0.015528696871662027, "replicates": 12, "sample_sd": 0.002275238560948795, "standard_error": 0.0006568047978172018 }, "log_loss": { "bootstrap_interval": [ -0.19153169402956677, -0.17694885495339305 ], "confidence_level": 0.95, "mean": -0.18376937580920363, "replicates": 12, "sample_sd": 0.0135915071058258, "standard_error": 0.003923530143120619 } } }, "discount": { "bootstrap_interval": [ 0.35025856936266175, 0.3680124102890438 ], "confidence_level": 0.95, "mean": 0.3594376421317226, "replicates": 12, "sample_sd": 0.016475146148551317, "standard_error": 0.004755965031902265 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.38010734956546466, "discount_min": 0.3300319111028459, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1690417879464527, 0.17092636410540143 ], "confidence_level": 0.95, "mean": 0.16989773111472198, "replicates": 12, "sample_sd": 0.0017615297107534298, "standard_error": 0.0005085098263445082 }, "log_loss": { "bootstrap_interval": [ 0.5228520464044778, 0.5268706065598348 ], "confidence_level": 0.95, "mean": 0.5247160628599736, "replicates": 12, "sample_sd": 0.0037341561368685093, "standard_error": 0.0010779580254085634 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.15618187923918386, 0.15844221156529445 ], "confidence_level": 0.95, "mean": 0.1572066299905949, "replicates": 12, "sample_sd": 0.0021003839509174723, "standard_error": 0.0006063286197318862 }, "log_loss": { "bootstrap_interval": [ 0.49193952240659755, 0.49676765942357703 ], "confidence_level": 0.95, "mean": 0.49415865696159383, "replicates": 12, "sample_sd": 0.004516209752183464, "standard_error": 0.001303717458069968 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.10546299906790313, 0.10991927036706685 ], "confidence_level": 0.95, "mean": 0.10762526277791494, "replicates": 12, "sample_sd": 0.004082773792405062, "standard_error": 0.001178595274042706 }, "log_loss": { "bootstrap_interval": [ 0.3382106775475636, 0.3490541044732385 ], "confidence_level": 0.95, "mean": 0.3434591512554226, "replicates": 12, "sample_sd": 0.009828376533981659, "standard_error": 0.0028372079187956563 } }, "independent": { "brier": { "bootstrap_interval": [ 0.13377795005884477, 0.14020895500841216 ], "confidence_level": 0.95, "mean": 0.1366346130420212, "replicates": 12, "sample_sd": 0.005996968062657086, "standard_error": 0.0017311755626483287 }, "log_loss": { "bootstrap_interval": [ 0.553910458986448, 0.5788774282657876 ], "confidence_level": 0.95, "mean": 0.5652981844094365, "replicates": 12, "sample_sd": 0.023214672926561104, "standard_error": 0.006701498831649586 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19857211991931484, 0.20212812421074755 ], "confidence_level": 0.95, "mean": 0.20024062340957027, "replicates": 12, "sample_sd": 0.0033193485060579353, "standard_error": 0.0009582133767533656 }, "log_loss": { "bootstrap_interval": [ 0.5811422411890791, 0.5890966759210486 ], "confidence_level": 0.95, "mean": 0.5849171285844086, "replicates": 12, "sample_sd": 0.0073727031483632865, "standard_error": 0.0021283160736813725 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11909448487726115, 0.12351722542298302 ], "confidence_level": 0.95, "mean": 0.12110591617035918, "replicates": 12, "sample_sd": 0.0040656020488031835, "standard_error": 0.001173638218647206 }, "log_loss": { "bootstrap_interval": [ 0.3765822619360943, 0.3877105316983354 ], "confidence_level": 0.95, "mean": 0.38152880860023286, "replicates": 12, "sample_sd": 0.010067448180691357, "standard_error": 0.002906221958587382 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.27027027027027023, 0.35714285714285715, 0.5263157894736842, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.35714285714285715, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-4-3-2-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.045044518673518025, -0.042799795102177525 ], "confidence_level": 0.95, "mean": -0.04413600356251732, "replicates": 12, "sample_sd": 0.0021233785669835686, "standard_error": 0.0006129665936197225 }, "log_loss": { "bootstrap_interval": [ -0.1321334641298138, -0.12602075647449593 ], "confidence_level": 0.95, "mean": -0.1294521116042439, "replicates": 12, "sample_sd": 0.0057357549976795865, "standard_error": 0.0016557698459580255 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.013868717664367243, 0.015732255201009047 ], "confidence_level": 0.95, "mean": 0.014812254267716759, "replicates": 12, "sample_sd": 0.0017493056559239304, "standard_error": 0.000504981045671308 }, "log_loss": { "bootstrap_interval": [ 0.03855018454087135, 0.043907481450368414 ], "confidence_level": 0.95, "mean": 0.04114783405473502, "replicates": 12, "sample_sd": 0.004974609778964857, "standard_error": 0.0014360461474993524 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.020029430171856794, -0.017285912513614186 ], "confidence_level": 0.95, "mean": -0.018518583732087302, "replicates": 12, "sample_sd": 0.0024898086441116846, "standard_error": 0.0007187458454542691 }, "log_loss": { "bootstrap_interval": [ -0.22585066609785506, -0.2100527944666827 ], "confidence_level": 0.95, "mean": -0.21711949929492613, "replicates": 12, "sample_sd": 0.014822002057379397, "standard_error": 0.004278743438878591 } } }, "discount": { "bootstrap_interval": [ 0.32725633000739157, 0.3457137517044942 ], "confidence_level": 0.95, "mean": 0.3365068958915568, "replicates": 12, "sample_sd": 0.016823725340467926, "standard_error": 0.004856591177045743 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3661370450733915, "discount_min": 0.3081336624641614, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16991796636560236, 0.1719160035740406 ], "confidence_level": 0.95, "mean": 0.17082482700293025, "replicates": 12, "sample_sd": 0.0018664900590881195, "standard_error": 0.0005388092690271432 }, "log_loss": { "bootstrap_interval": [ 0.5242951411817999, 0.528544441264339 ], "confidence_level": 0.95, "mean": 0.5262384508692486, "replicates": 12, "sample_sd": 0.003963261665544133, "standard_error": 0.001144095094735415 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.16254621838440575, 0.1646643057691446 ], "confidence_level": 0.95, "mean": 0.16351900526143717, "replicates": 12, "sample_sd": 0.0020272669252926227, "standard_error": 0.0005852215525184604 }, "log_loss": { "bootstrap_interval": [ 0.5067750252205345, 0.5115126213443624 ], "confidence_level": 0.95, "mean": 0.5089049259946871, "replicates": 12, "sample_sd": 0.004342623976738327, "standard_error": 0.0012536075609795982 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.109847034254824, 0.11407691514214957 ], "confidence_level": 0.95, "mean": 0.11187656917269617, "replicates": 12, "sample_sd": 0.003957125877630357, "standard_error": 0.0011423238453335605 }, "log_loss": { "bootstrap_interval": [ 0.35050020643884866, 0.36134410317344273 ], "confidence_level": 0.95, "mean": 0.35563850521026963, "replicates": 12, "sample_sd": 0.01019728689096922, "standard_error": 0.0029437031657524606 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14248644586039969, 0.14881915907593307 ], "confidence_level": 0.95, "mean": 0.14520740717250022, "replicates": 12, "sample_sd": 0.0060157074801834515, "standard_error": 0.0017365851665249806 }, "log_loss": { "bootstrap_interval": [ 0.6028852816442796, 0.6279698187355784 ], "confidence_level": 0.95, "mean": 0.6139058385599307, "replicates": 12, "sample_sd": 0.023488534381505645, "standard_error": 0.006780555824016032 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19818894080364624, 0.20204197739741342 ], "confidence_level": 0.95, "mean": 0.19992742603064345, "replicates": 12, "sample_sd": 0.0035618422896455063, "standard_error": 0.0010282153023689132 }, "log_loss": { "bootstrap_interval": [ 0.5797948570712517, 0.58847799733925 ], "confidence_level": 0.95, "mean": 0.5838355487280672, "replicates": 12, "sample_sd": 0.007996384601348563, "standard_error": 0.002308357401066186 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12494569598713338, 0.12903298747879235 ], "confidence_level": 0.95, "mean": 0.12668882344041293, "replicates": 12, "sample_sd": 0.0038573337585458677, "standard_error": 0.0011135163419253439 }, "log_loss": { "bootstrap_interval": [ 0.3923691685247759, 0.4025392862245813 ], "confidence_level": 0.95, "mean": 0.3967863392650046, "replicates": 12, "sample_sd": 0.009453000344209097, "standard_error": 0.002728846146689374 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25, 0.3333333333333333, 0.5, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-5-5-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12166921693042292, -0.11950714260905676 ], "confidence_level": 0.95, "mean": -0.12064721933313702, "replicates": 12, "sample_sd": 0.001984274791145078, "standard_error": 0.0005728107924068996 }, "log_loss": { "bootstrap_interval": [ -0.37505891764438526, -0.3678412554746731 ], "confidence_level": 0.95, "mean": -0.3716041505299348, "replicates": 12, "sample_sd": 0.006819939578562517, "standard_error": 0.0019687469757700264 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 1.4637011269911723e-05, 0.0001289766614240679 ], "confidence_level": 0.95, "mean": 6.599135730207238e-05, "replicates": 12, "sample_sd": 0.00010642515210320103, "standard_error": 3.072229510766499e-05 }, "log_loss": { "bootstrap_interval": [ 0.00021999171800967955, 0.0014172818975749554 ], "confidence_level": 0.95, "mean": 0.0007536849467200841, "replicates": 12, "sample_sd": 0.0011100584009282341, "standard_error": 0.00032044625829606076 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 1.4637011269909411e-05, 0.00012897666142407018 ], "confidence_level": 0.95, "mean": 6.599135730207123e-05, "replicates": 12, "sample_sd": 0.00010642515210320401, "standard_error": 3.072229510766585e-05 }, "log_loss": { "bootstrap_interval": [ 0.00021999171800967963, 0.0014172818975749554 ], "confidence_level": 0.95, "mean": 0.0007536849467200865, "replicates": 12, "sample_sd": 0.0011100584009282324, "standard_error": 0.00032044625829606027 } } }, "discount": { "bootstrap_interval": [ 0.9145581883836195, 0.9752157647526639 ], "confidence_level": 0.95, "mean": 0.9466173145612724, "replicates": 12, "sample_sd": 0.05707188629628911, "standard_error": 0.016475234458161115 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8477297425356989, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16159036506198657, 0.1628962844559556 ], "confidence_level": 0.95, "mean": 0.16229312162995393, "replicates": 12, "sample_sd": 0.001206242253059362, "standard_error": 0.0003482121447558617 }, "log_loss": { "bootstrap_interval": [ 0.5114943079907625, 0.5141657615251269 ], "confidence_level": 0.95, "mean": 0.5129265246198856, "replicates": 12, "sample_sd": 0.002501743789771024, "standard_error": 0.0007221912252338877 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.040081232413498064, 0.04311172053968773 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712502, "standard_error": 0.0008155893598404593 }, "log_loss": { "bootstrap_interval": [ 0.13590487095895593, 0.14542739302169402 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.04006087991612155, 0.04312762894441603 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.0028252844187125017, "standard_error": 0.0008155893598404591 }, "log_loss": { "bootstrap_interval": [ 0.13587749747502995, 0.14545530436952556 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04010105360831567, 0.04311667633840343 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712502, "standard_error": 0.0008155893598404593 }, "log_loss": { "bootstrap_interval": [ 0.13591083601523773, 0.14548925968637078 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19832185331679422, 0.20299702953164014 ], "confidence_level": 0.95, "mean": 0.20076021787067505, "replicates": 12, "sample_sd": 0.004305894758847835, "standard_error": 0.0012430047490614982 }, "log_loss": { "bootstrap_interval": [ 0.5802521340773237, 0.5909438676657157 ], "confidence_level": 0.95, "mean": 0.5857046061562484, "replicates": 12, "sample_sd": 0.009804608943974726, "standard_error": 0.0028303468065514104 } }, "learned": { "brier": { "bootstrap_interval": [ 0.040162402028605715, 0.043171411592082216 ], "confidence_level": 0.95, "mean": 0.04164590229681692, "replicates": 12, "sample_sd": 0.002810023435715211, "standard_error": 0.000811183893519667 }, "log_loss": { "bootstrap_interval": [ 0.13679971973286942, 0.14605243057956308 ], "confidence_level": 0.95, "mean": 0.14132237408995083, "replicates": 12, "sample_sd": 0.008621266070041886, "standard_error": 0.002488745143147035 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-5-5-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04559190969638921, -0.04259050853653161 ], "confidence_level": 0.95, "mean": -0.04415066687792214, "replicates": 12, "sample_sd": 0.002715079237828494, "standard_error": 0.0007837758644157225 }, "log_loss": { "bootstrap_interval": [ -0.1339313017852147, -0.12604847243907916 ], "confidence_level": 0.95, "mean": -0.1301232033776712, "replicates": 12, "sample_sd": 0.007309885752911432, "standard_error": 0.0021101822535944128 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -3.9835924409528146e-05, 0.00014196112898008732 ], "confidence_level": 0.95, "mean": 4.6941787438751353e-05, "replicates": 12, "sample_sd": 0.00016823040949292458, "standard_error": 4.856393610331049e-05 }, "log_loss": { "bootstrap_interval": [ 2.360160151926537e-05, 0.0007144081498349354 ], "confidence_level": 0.95, "mean": 0.0003396290063489571, "replicates": 12, "sample_sd": 0.0006387519466399147, "standard_error": 0.00018439180416897612 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.01992876222624478, -0.01704466944671381 ], "confidence_level": 0.95, "mean": -0.01841003277200371, "replicates": 12, "sample_sd": 0.0026399016217731525, "standard_error": 0.0007620739559824297 }, "log_loss": { "bootstrap_interval": [ -0.2249108776621697, -0.20474373594998957 ], "confidence_level": 0.95, "mean": -0.21482106137585935, "replicates": 12, "sample_sd": 0.01863600057951382, "standard_error": 0.005379749975600163 } } }, "discount": { "bootstrap_interval": [ 0.3242746177823957, 0.34649954402986044 ], "confidence_level": 0.95, "mean": 0.3345446566951045, "replicates": 12, "sample_sd": 0.02046054525103088, "standard_error": 0.0059064506542245996 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3753818416755297, "discount_min": 0.30766862935266015, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16924471576961758, 0.1711744250031224 ], "confidence_level": 0.95, "mean": 0.17015470214840595, "replicates": 12, "sample_sd": 0.0017798277245558766, "standard_error": 0.0005137920079417472 }, "log_loss": { "bootstrap_interval": [ 0.5229392221834848, 0.5269973424952159 ], "confidence_level": 0.95, "mean": 0.5248518794905525, "replicates": 12, "sample_sd": 0.003766907252981392, "standard_error": 0.0010874124582605801 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.13478553239920102, 0.13800901270751387 ], "confidence_level": 0.95, "mean": 0.1363392898828836, "replicates": 12, "sample_sd": 0.003023914421219431, "standard_error": 0.0008729289025487149 }, "log_loss": { "bootstrap_interval": [ 0.4301384160736085, 0.43737758355459316 ], "confidence_level": 0.95, "mean": 0.4335392898035929, "replicates": 12, "sample_sd": 0.006793441565124474, "standard_error": 0.001961097658174304 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.12371968954770127, 0.12838369071490793 ], "confidence_level": 0.95, "mean": 0.12595709348304507, "replicates": 12, "sample_sd": 0.004363288823243466, "standard_error": 0.0012595729883258504 }, "log_loss": { "bootstrap_interval": [ 0.3886036345233267, 0.4006543522769742 ], "confidence_level": 0.95, "mean": 0.39438904710653233, "replicates": 12, "sample_sd": 0.011188728887599088, "standard_error": 0.003229907817572538 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14116076514635564, 0.1480196073012229 ], "confidence_level": 0.95, "mean": 0.14441406804248752, "replicates": 12, "sample_sd": 0.006364570497980392, "standard_error": 0.001837293245142665 }, "log_loss": { "bootstrap_interval": [ 0.5938966942548098, 0.6252752697556226 ], "confidence_level": 0.95, "mean": 0.6095497374887406, "replicates": 12, "sample_sd": 0.02912899980076636, "standard_error": 0.008408817938098506 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1985736230422999, 0.2022896103809159 ], "confidence_level": 0.95, "mean": 0.20051126014712403, "replicates": 12, "sample_sd": 0.0034439886434354883, "standard_error": 0.0009941938851867466 }, "log_loss": { "bootstrap_interval": [ 0.5814348720777042, 0.5898284422466582 ], "confidence_level": 0.95, "mean": 0.5857271248029461, "replicates": 12, "sample_sd": 0.007748180206795777, "standard_error": 0.0022367069640616363 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12376165006665336, 0.12844475275518866 ], "confidence_level": 0.95, "mean": 0.12600403527048382, "replicates": 12, "sample_sd": 0.004319896511195425, "standard_error": 0.0012470467068050018 }, "log_loss": { "bootstrap_interval": [ 0.38907624928693246, 0.40083532698845664 ], "confidence_level": 0.95, "mean": 0.3947286761128813, "replicates": 12, "sample_sd": 0.010882189800265679, "standard_error": 0.003141417605277995 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333, 0.3333333333333333 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-5-5-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.020829294815470654, -0.018881827420002655 ], "confidence_level": 0.95, "mean": -0.0199024951919456, "replicates": 12, "sample_sd": 0.001810908807569638, "standard_error": 0.0005227643437640974 }, "log_loss": { "bootstrap_interval": [ -0.06141918826449352, -0.055897874418668864 ], "confidence_level": 0.95, "mean": -0.058808673278246695, "replicates": 12, "sample_sd": 0.005272041834752149, "standard_error": 0.001521907386236561 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -2.4754171835452675e-06, 0.00015870182564316305 ], "confidence_level": 0.95, "mean": 6.78473351432123e-05, "replicates": 12, "sample_sd": 0.00014776166289947297, "standard_error": 4.265511792545873e-05 }, "log_loss": { "bootstrap_interval": [ 5.7309623772985985e-05, 0.0004413957298303507 ], "confidence_level": 0.95, "mean": 0.00024700927710028425, "replicates": 12, "sample_sd": 0.0003548573934332677, "standard_error": 0.00010243850581131304 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.038966390800907504, -0.034722301919099446 ], "confidence_level": 0.95, "mean": -0.03671804062657634, "replicates": 12, "sample_sd": 0.003879659344294475, "standard_error": 0.0011199611833962312 }, "log_loss": { "bootstrap_interval": [ -0.5150738627964062, -0.4821162229865448 ], "confidence_level": 0.95, "mean": -0.49762157992211675, "replicates": 12, "sample_sd": 0.031348050189184275, "standard_error": 0.00904940260764772 } } }, "discount": { "bootstrap_interval": [ 0.21141850989643995, 0.2260876634999599 ], "confidence_level": 0.95, "mean": 0.2188028943285033, "replicates": 12, "sample_sd": 0.013553261707611805, "standard_error": 0.003912489647643561 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.23920774383208748, "discount_min": 0.19854440987832023, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1756432719841883, 0.17813475897774536 ], "confidence_level": 0.95, "mean": 0.17677007750607426, "replicates": 12, "sample_sd": 0.002285864530789333, "standard_error": 0.0006598722510911195 }, "log_loss": { "bootstrap_interval": [ 0.5334713392464988, 0.5389118232177713 ], "confidence_level": 0.95, "mean": 0.5359691062511771, "replicates": 12, "sample_sd": 0.004981006046221678, "standard_error": 0.0014378925908106197 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1679842327834417, 0.170795782650254 ], "confidence_level": 0.95, "mean": 0.1692779550545317, "replicates": 12, "sample_sd": 0.0026031074089639438, "standard_error": 0.0007514523816474211 }, "log_loss": { "bootstrap_interval": [ 0.5136505963221207, 0.5198346544808132 ], "confidence_level": 0.95, "mean": 0.5165363147874866, "replicates": 12, "sample_sd": 0.005783393116746974, "standard_error": 0.001669521786391647 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.1548354215849662, 0.1592057339176697 ], "confidence_level": 0.95, "mean": 0.15679973497898547, "replicates": 12, "sample_sd": 0.004050412940396135, "standard_error": 0.0011692535007334262 }, "log_loss": { "bootstrap_interval": [ 0.4718098124324486, 0.48295809871444684 ], "confidence_level": 0.95, "mean": 0.4769134236958301, "replicates": 12, "sample_sd": 0.010267322188356787, "standard_error": 0.0029639206146522047 } }, "independent": { "brier": { "bootstrap_interval": [ 0.18987665720746283, 0.19790051752314292 ], "confidence_level": 0.95, "mean": 0.19358562294070503, "replicates": 12, "sample_sd": 0.007467228540420739, "standard_error": 0.002155603203956185 }, "log_loss": { "bootstrap_interval": [ 0.9539246614520325, 0.998729499579276 ], "confidence_level": 0.95, "mean": 0.9747820128950472, "replicates": 12, "sample_sd": 0.04115994751077374, "standard_error": 0.011881853387588043 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19858731767431725, 0.2021202205172172 ], "confidence_level": 0.95, "mean": 0.20024062340957027, "replicates": 12, "sample_sd": 0.0033193485060579353, "standard_error": 0.0009582133767533656 }, "log_loss": { "bootstrap_interval": [ 0.5810338276459351, 0.5891223565970456 ], "confidence_level": 0.95, "mean": 0.5849171285844086, "replicates": 12, "sample_sd": 0.0073727031483632865, "standard_error": 0.0021283160736813725 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15484985508226187, 0.15917344795388075 ], "confidence_level": 0.95, "mean": 0.15686758231412867, "replicates": 12, "sample_sd": 0.003993407372083611, "standard_error": 0.0011527974106281546 }, "log_loss": { "bootstrap_interval": [ 0.47200070884422796, 0.4829727492185845 ], "confidence_level": 0.95, "mean": 0.4771604329729304, "replicates": 12, "sample_sd": 0.010097121075805444, "standard_error": 0.0029147877855782585 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2173913043478261, 0.2173913043478261 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-5-5-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.01693235355636167, -0.015252011149887525 ], "confidence_level": 0.95, "mean": -0.016130679250106344, "replicates": 12, "sample_sd": 0.0015741697572638042, "standard_error": 0.00045442366655321266 }, "log_loss": { "bootstrap_interval": [ -0.05036654174650655, -0.04502775904671679 ], "confidence_level": 0.95, "mean": -0.04783929757229083, "replicates": 12, "sample_sd": 0.004868254205941648, "standard_error": 0.0014053439381419694 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 1.4623400911531018e-05, 0.00017415230810056945 ], "confidence_level": 0.95, "mean": 8.760171372317281e-05, "replicates": 12, "sample_sd": 0.00014764866035855796, "standard_error": 4.262249690175053e-05 }, "log_loss": { "bootstrap_interval": [ 5.2512311716474804e-05, 0.0005282388934733475 ], "confidence_level": 0.95, "mean": 0.00027818281892777924, "replicates": 12, "sample_sd": 0.00043859599942065096, "standard_error": 0.0001266117591655029 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.043260377749012736, -0.03887437554830714 ], "confidence_level": 0.95, "mean": -0.04092511337994088, "replicates": 12, "sample_sd": 0.004161637390745185, "standard_error": 0.001201361233908172 }, "log_loss": { "bootstrap_interval": [ -0.5921098879202881, -0.5538212811789516 ], "confidence_level": 0.95, "mean": -0.5714148793238729, "replicates": 12, "sample_sd": 0.035118979622078335, "standard_error": 0.010137976169235955 } } }, "discount": { "bootstrap_interval": [ 0.19466971454073936, 0.20884880572270775 ], "confidence_level": 0.95, "mean": 0.2017442263544697, "replicates": 12, "sample_sd": 0.013098010788515482, "standard_error": 0.003781070027299018 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.22091790658313465, "discount_min": 0.18033642687320367, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17731345742810806, 0.1799641178018061 ], "confidence_level": 0.95, "mean": 0.178545283323189, "replicates": 12, "sample_sd": 0.002445727737313546, "standard_error": 0.0007060207837512551 }, "log_loss": { "bootstrap_interval": [ 0.5364917877965603, 0.5423097993710841 ], "confidence_level": 0.95, "mean": 0.5391394753981947, "replicates": 12, "sample_sd": 0.005365260829323291, "standard_error": 0.0015488173920411785 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.17357182512352348, 0.17630516128432208 ], "confidence_level": 0.95, "mean": 0.17482922771735868, "replicates": 12, "sample_sd": 0.002554339123976425, "standard_error": 0.000737374190414691 }, "log_loss": { "bootstrap_interval": [ 0.5269121171314899, 0.533052413308761 ], "confidence_level": 0.95, "mean": 0.5297750680875686, "replicates": 12, "sample_sd": 0.0056810438387898355, "standard_error": 0.0016399760948016882 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.16040102204689666, 0.16463953784007093 ], "confidence_level": 0.95, "mean": 0.16232700235935948, "replicates": 12, "sample_sd": 0.003944466077089554, "standard_error": 0.0011386692757085007 }, "log_loss": { "bootstrap_interval": [ 0.48610347569957296, 0.4968849369472076 ], "confidence_level": 0.95, "mean": 0.4910219950069761, "replicates": 12, "sample_sd": 0.010013409517682292, "standard_error": 0.0028906223402699162 } }, "independent": { "brier": { "bootstrap_interval": [ 0.19961273971466406, 0.20775190850763617 ], "confidence_level": 0.95, "mean": 0.20333971745302354, "replicates": 12, "sample_sd": 0.007761201121982291, "standard_error": 0.0022404657785056506 }, "log_loss": { "bootstrap_interval": [ 1.04024838187257, 1.0885392927686663 ], "confidence_level": 0.95, "mean": 1.0627150571497768, "replicates": 12, "sample_sd": 0.044749729562791433, "standard_error": 0.012918134204620294 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19815240876344747, 0.2020132369742114 ], "confidence_level": 0.95, "mean": 0.19992742603064345, "replicates": 12, "sample_sd": 0.0035618422896455063, "standard_error": 0.0010282153023689132 }, "log_loss": { "bootstrap_interval": [ 0.5798656512406192, 0.588446535709538 ], "confidence_level": 0.95, "mean": 0.5838355487280672, "replicates": 12, "sample_sd": 0.007996384601348563, "standard_error": 0.002308357401066186 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16046397023792638, 0.16468230999816402 ], "confidence_level": 0.95, "mean": 0.16241460407308267, "replicates": 12, "sample_sd": 0.003909824271984771, "standard_error": 0.00112866904795727 }, "log_loss": { "bootstrap_interval": [ 0.4863760120585988, 0.49703969049645774 ], "confidence_level": 0.95, "mean": 0.49130017782590385, "replicates": 12, "sample_sd": 0.009946797357016592, "standard_error": 0.002871393065824094 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 0.2 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-9-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12163913885952571, -0.11953525174335498 ], "confidence_level": 0.95, "mean": -0.12064721933313702, "replicates": 12, "sample_sd": 0.001984274791145078, "standard_error": 0.0005728107924068996 }, "log_loss": { "bootstrap_interval": [ -0.3751011833130093, -0.3677420304532883 ], "confidence_level": 0.95, "mean": -0.3716041505299348, "replicates": 12, "sample_sd": 0.006819939578562517, "standard_error": 0.0019687469757700264 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 1.4331723579993895e-05, 0.00012961959339642182 ], "confidence_level": 0.95, "mean": 6.599135730207238e-05, "replicates": 12, "sample_sd": 0.00010642515210320216, "standard_error": 3.0722295107665316e-05 }, "log_loss": { "bootstrap_interval": [ 0.00021796105010137208, 0.0014203748339730196 ], "confidence_level": 0.95, "mean": 0.0007536849467200841, "replicates": 12, "sample_sd": 0.0011100584009282341, "standard_error": 0.00032044625829606076 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 1.43317235799921e-05, 0.00012961959339642125 ], "confidence_level": 0.95, "mean": 6.599135730207123e-05, "replicates": 12, "sample_sd": 0.00010642515210320401, "standard_error": 3.072229510766585e-05 }, "log_loss": { "bootstrap_interval": [ 0.00021796105010137433, 0.0014203748339730196 ], "confidence_level": 0.95, "mean": 0.0007536849467200865, "replicates": 12, "sample_sd": 0.0011100584009282324, "standard_error": 0.00032044625829606027 } } }, "discount": { "bootstrap_interval": [ 0.914340073312451, 0.9765654909240418 ], "confidence_level": 0.95, "mean": 0.9466173145612724, "replicates": 12, "sample_sd": 0.05707188629628911, "standard_error": 0.016475234458161115 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.8477297425356989, "discount_one_count": 5, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1616065692786548, 0.16290886231235838 ], "confidence_level": 0.95, "mean": 0.16229312162995393, "replicates": 12, "sample_sd": 0.001206242253059362, "standard_error": 0.0003482121447558617 }, "log_loss": { "bootstrap_interval": [ 0.5114944192670423, 0.5141568760527077 ], "confidence_level": 0.95, "mean": 0.5129265246198856, "replicates": 12, "sample_sd": 0.002501743789771024, "standard_error": 0.0007221912252338877 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.04009447301589451, 0.04313863367125509 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712502, "standard_error": 0.0008155893598404593 }, "log_loss": { "bootstrap_interval": [ 0.13581956120942995, 0.14549029560706592 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.04007215346108587, 0.04310740157266905 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712503, "standard_error": 0.0008155893598404595 }, "log_loss": { "bootstrap_interval": [ 0.1359075817922953, 0.14544499770070918 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04007313067636663, 0.043171156708360296 ], "confidence_level": 0.95, "mean": 0.041579910939514846, "replicates": 12, "sample_sd": 0.002825284418712502, "standard_error": 0.0008155893598404593 }, "log_loss": { "bootstrap_interval": [ 0.13579954607049458, 0.14544300967474927 ], "confidence_level": 0.95, "mean": 0.14056868914323076, "replicates": 12, "sample_sd": 0.008857977703056696, "standard_error": 0.002557077905667743 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19833014321036818, 0.2029583131193876 ], "confidence_level": 0.95, "mean": 0.20076021787067505, "replicates": 12, "sample_sd": 0.004305894758847835, "standard_error": 0.0012430047490614982 }, "log_loss": { "bootstrap_interval": [ 0.580196483684072, 0.5908499058486922 ], "confidence_level": 0.95, "mean": 0.5857046061562484, "replicates": 12, "sample_sd": 0.009804608943974726, "standard_error": 0.0028303468065514104 } }, "learned": { "brier": { "bootstrap_interval": [ 0.040130355938693026, 0.04320847990579706 ], "confidence_level": 0.95, "mean": 0.04164590229681692, "replicates": 12, "sample_sd": 0.002810023435715211, "standard_error": 0.000811183893519667 }, "log_loss": { "bootstrap_interval": [ 0.1368323249683863, 0.14615625523318573 ], "confidence_level": 0.95, "mean": 0.14132237408995083, "replicates": 12, "sample_sd": 0.008621266070041886, "standard_error": 0.002488745143147035 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-9-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.020114843278892767, -0.018411900353604755 ], "confidence_level": 0.95, "mean": -0.019278169267718272, "replicates": 12, "sample_sd": 0.0015697612174038608, "standard_error": 0.00045315103071577687 }, "log_loss": { "bootstrap_interval": [ -0.05937060496986801, -0.0544911717070587 ], "confidence_level": 0.95, "mean": -0.05696823408845057, "replicates": 12, "sample_sd": 0.004529616047343838, "standard_error": 0.0013075875221298069 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.017213711898227665, 0.020681792152887932 ], "confidence_level": 0.95, "mean": 0.019135174486572922, "replicates": 12, "sample_sd": 0.00315455672634622, "standard_error": 0.0009106420875649673 }, "log_loss": { "bootstrap_interval": [ 0.04566364421357445, 0.054513477512975354 ], "confidence_level": 0.95, "mean": 0.050609733780272045, "replicates": 12, "sample_sd": 0.008266759445576567, "standard_error": 0.0023864078956147565 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03899386915462383, -0.03608412187226162 ], "confidence_level": 0.95, "mean": -0.03739612253127634, "replicates": 12, "sample_sd": 0.0027185829561949875, "standard_error": 0.0007847873007867524 }, "log_loss": { "bootstrap_interval": [ -0.5269292132214776, -0.4946227299481715 ], "confidence_level": 0.95, "mean": -0.5110148533513519, "replicates": 12, "sample_sd": 0.029999142803638273, "standard_error": 0.008660006586569291 } } }, "discount": { "bootstrap_interval": [ 0.21073288958307462, 0.2225598959901255 ], "confidence_level": 0.95, "mean": 0.21651623735380082, "replicates": 12, "sample_sd": 0.010986282888277132, "standard_error": 0.003171466691470091 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.23566539597985675, "discount_min": 0.19750437824534933, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1761023566039855, 0.1784704404816967 ], "confidence_level": 0.95, "mean": 0.17717895990267926, "replicates": 12, "sample_sd": 0.0022035253298794285, "standard_error": 0.0006361029711860235 }, "log_loss": { "bootstrap_interval": [ 0.5347193255237858, 0.5397769180453392 ], "confidence_level": 0.95, "mean": 0.5370632178034394, "replicates": 12, "sample_sd": 0.004769226812508416, "standard_error": 0.0013767571920140576 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1569246094083169, 0.1606237405231728 ], "confidence_level": 0.95, "mean": 0.15864182403286273, "replicates": 12, "sample_sd": 0.0034066491073077763, "standard_error": 0.0009834148895693715 }, "log_loss": { "bootstrap_interval": [ 0.47877601195251535, 0.48763081071036146 ], "confidence_level": 0.95, "mean": 0.4829637315056138, "replicates": 12, "sample_sd": 0.008231252303241052, "standard_error": 0.0023761578665219742 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.13636351351697099, 0.14120104391695135 ], "confidence_level": 0.95, "mean": 0.13876561614838806, "replicates": 12, "sample_sd": 0.004413891271155556, "standard_error": 0.0012741806567877 }, "log_loss": { "bootstrap_interval": [ 0.4242333641005091, 0.43502193715053267 ], "confidence_level": 0.95, "mean": 0.4294852499347167, "replicates": 12, "sample_sd": 0.010140106660601632, "standard_error": 0.0029271966550549347 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1923965837767594, 0.19877860624095564 ], "confidence_level": 0.95, "mean": 0.19529691316623732, "replicates": 12, "sample_sd": 0.005922789693806417, "standard_error": 0.0017097621120363382 }, "log_loss": { "bootstrap_interval": [ 0.9701403752653827, 1.0116837032231585 ], "confidence_level": 0.95, "mean": 0.9911098370663405, "replicates": 12, "sample_sd": 0.038437425797066334, "standard_error": 0.011095929065446257 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19862491798340093, 0.20234627479628617 ], "confidence_level": 0.95, "mean": 0.20051126014712403, "replicates": 12, "sample_sd": 0.0034439886434354883, "standard_error": 0.0009941938851867466 }, "log_loss": { "bootstrap_interval": [ 0.5814047507032097, 0.5898495521518647 ], "confidence_level": 0.95, "mean": 0.5857271248029461, "replicates": 12, "sample_sd": 0.007748180206795777, "standard_error": 0.0022367069640616363 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15605433680657277, 0.16003171703065985 ], "confidence_level": 0.95, "mean": 0.15790079063496099, "replicates": 12, "sample_sd": 0.0036930666913728345, "standard_error": 0.0010660965241996734 }, "log_loss": { "bootstrap_interval": [ 0.47538134131211696, 0.48520014310359616 ], "confidence_level": 0.95, "mean": 0.4800949837149887, "replicates": 12, "sample_sd": 0.009117009543006671, "standard_error": 0.0026318539569296444 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-9-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.003092722422427458, -0.0024192046902226276 ], "confidence_level": 0.95, "mean": -0.0028014800596932268, "replicates": 12, "sample_sd": 0.0006260696053682351, "standard_error": 0.00018073072759539665 }, "log_loss": { "bootstrap_interval": [ -0.00900657265692199, -0.007249195278677732 ], "confidence_level": 0.95, "mean": -0.00818561886958119, "replicates": 12, "sample_sd": 0.0016039320632182865, "standard_error": 0.0004630153042304748 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.025537951348326732, 0.02916905303312848 ], "confidence_level": 0.95, "mean": 0.02760516931410186, "replicates": 12, "sample_sd": 0.0033884582121726774, "standard_error": 0.0009781636304678468 }, "log_loss": { "bootstrap_interval": [ 0.06431518542095144, 0.07354729676444006 ], "confidence_level": 0.95, "mean": 0.06948930936392089, "replicates": 12, "sample_sd": 0.008631264226110201, "standard_error": 0.002491631362195756 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06542798182999808, -0.060067463459107844 ], "confidence_level": 0.95, "mean": -0.06263439917475556, "replicates": 12, "sample_sd": 0.005035067533698697, "standard_error": 0.001453498797984444 }, "log_loss": { "bootstrap_interval": [ -1.0584659559622653, -1.0117851163031284 ], "confidence_level": 0.95, "mean": -1.0348579868043992, "replicates": 12, "sample_sd": 0.04274711972521256, "standard_error": 0.012340030540216316 } } }, "discount": { "bootstrap_interval": [ 0.12862805863284077, 0.13776364132080401 ], "confidence_level": 0.95, "mean": 0.13344402353880602, "replicates": 12, "sample_sd": 0.00838236625165422, "standard_error": 0.0024197807059192994 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.14419457580700185, "discount_min": 0.11495231063577255, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18785091100377244, 0.19102383340884735 ], "confidence_level": 0.95, "mean": 0.18926594589143847, "replicates": 12, "sample_sd": 0.002955522530031857, "standard_error": 0.0008531858641549482 }, "log_loss": { "bootstrap_interval": [ 0.5568358299954879, 0.5638052301925174 ], "confidence_level": 0.95, "mean": 0.5599435140421214, "replicates": 12, "sample_sd": 0.006541722555338011, "standard_error": 0.0018884326391441237 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.18569790695526675, 0.18911899893552594 ], "confidence_level": 0.95, "mean": 0.18721652225342728, "replicates": 12, "sample_sd": 0.0032077067879600837, "standard_error": 0.0009259851887550722 }, "log_loss": { "bootstrap_interval": [ 0.5507137189783828, 0.5585789001743928 ], "confidence_level": 0.95, "mean": 0.5542494186216359, "replicates": 12, "sample_sd": 0.007223827224278842, "standard_error": 0.0020853392962583684 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.1566924193993914, 0.16129382372715045 ], "confidence_level": 0.95, "mean": 0.15885929651764338, "replicates": 12, "sample_sd": 0.004221287911511605, "standard_error": 0.0012185808560190691 }, "log_loss": { "bootstrap_interval": [ 0.47713932030358724, 0.4880761305489174 ], "confidence_level": 0.95, "mean": 0.4822685858086193, "replicates": 12, "sample_sd": 0.010192045602182983, "standard_error": 0.0029421901360066434 } }, "independent": { "brier": { "bootstrap_interval": [ 0.24523895874931906, 0.25382909637788426 ], "confidence_level": 0.95, "mean": 0.2490988650065008, "replicates": 12, "sample_sd": 0.007908398487372502, "standard_error": 0.0022829579977716717 }, "log_loss": { "bootstrap_interval": [ 1.560347317960136, 1.614095146609723 ], "confidence_level": 0.95, "mean": 1.5866158819769396, "replicates": 12, "sample_sd": 0.05016859309214218, "standard_error": 0.014482425363306545 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1985806370012641, 0.20219635049845622 ], "confidence_level": 0.95, "mean": 0.20024062340957027, "replicates": 12, "sample_sd": 0.0033193485060579353, "standard_error": 0.0009582133767533656 }, "log_loss": { "bootstrap_interval": [ 0.581057841023334, 0.5891611942439519 ], "confidence_level": 0.95, "mean": 0.5849171285844086, "replicates": 12, "sample_sd": 0.0073727031483632865, "standard_error": 0.0021283160736813725 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18483473964973682, 0.1885804796355374 ], "confidence_level": 0.95, "mean": 0.18646446583174525, "replicates": 12, "sample_sd": 0.003512302333255332, "standard_error": 0.0010139143487901583 }, "log_loss": { "bootstrap_interval": [ 0.5478635235984635, 0.5565413614798136 ], "confidence_level": 0.95, "mean": 0.5517578951725403, "replicates": 12, "sample_sd": 0.008003860559493819, "standard_error": 0.002310515524289993 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12195121951219513, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.13368983957219252, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 }, "groups-9-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0015030600860824232, -0.0010703391054741565 ], "confidence_level": 0.95, "mean": -0.0013173108282528944, "replicates": 12, "sample_sd": 0.0004075928934665611, "standard_error": 0.00011766193338134877 }, "log_loss": { "bootstrap_interval": [ -0.004032152131512739, -0.0029067852393976808 ], "confidence_level": 0.95, "mean": -0.0035111759971049685, "replicates": 12, "sample_sd": 0.0010335403485244353, "standard_error": 0.0002983573992194612 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.027023349862731673, 0.030128755897403568 ], "confidence_level": 0.95, "mean": 0.028755652450261005, "replicates": 12, "sample_sd": 0.002896376954888946, "standard_error": 0.0008361120072898808 }, "log_loss": { "bootstrap_interval": [ 0.06739669973306123, 0.0755708167324024 ], "confidence_level": 0.95, "mean": 0.07183604527134474, "replicates": 12, "sample_sd": 0.007438017033256936, "standard_error": 0.002147170568193957 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.07111269888625099, -0.06434199048438273 ], "confidence_level": 0.95, "mean": -0.06766913732198523, "replicates": 12, "sample_sd": 0.0062076707479948885, "standard_error": 0.0017920001886977074 }, "log_loss": { "bootstrap_interval": [ -1.1880959687412125, -1.1385919304609835 ], "confidence_level": 0.95, "mean": -1.1618026967649033, "replicates": 12, "sample_sd": 0.04471314809190132, "standard_error": 0.012907574043587415 } } }, "discount": { "bootstrap_interval": [ 0.1172647749658251, 0.126050746811982 ], "confidence_level": 0.95, "mean": 0.12196945146368518, "replicates": 12, "sample_sd": 0.008172180783211098, "standard_error": 0.002359105387526607 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.1307671690146201, "discount_min": 0.10342372696015945, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19093990989018964, 0.19427353382841758 ], "confidence_level": 0.95, "mean": 0.19239996563787337, "replicates": 12, "sample_sd": 0.003164539794864637, "standard_error": 0.0009135239512131906 }, "log_loss": { "bootstrap_interval": [ 0.5630659554117299, 0.5705238667808795 ], "confidence_level": 0.95, "mean": 0.5663692162754258, "replicates": 12, "sample_sd": 0.007020056457689224, "standard_error": 0.002026515742786622 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.19031065779823012, 0.19365892285978992 ], "confidence_level": 0.95, "mean": 0.19179482818678187, "replicates": 12, "sample_sd": 0.0031757202799906994, "standard_error": 0.0009167514792617921 }, "log_loss": { "bootstrap_interval": [ 0.5618148492638806, 0.5693164440486986 ], "confidence_level": 0.95, "mean": 0.5651918916969032, "replicates": 12, "sample_sd": 0.007037121085917382, "standard_error": 0.002031441876637196 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.16036174005740436, 0.16460486915991837 ], "confidence_level": 0.95, "mean": 0.16232700235935948, "replicates": 12, "sample_sd": 0.003944466077089555, "standard_error": 0.0011386692757085009 }, "log_loss": { "bootstrap_interval": [ 0.48597696111301497, 0.49670193467461926 ], "confidence_level": 0.95, "mean": 0.4910219950069761, "replicates": 12, "sample_sd": 0.010013409517682292, "standard_error": 0.0028906223402699162 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2541773112361986, 0.2638692539059905 ], "confidence_level": 0.95, "mean": 0.2587517921316057, "replicates": 12, "sample_sd": 0.008945297850579199, "standard_error": 0.002582285061006641 }, "log_loss": { "bootstrap_interval": [ 1.6979029453723744, 1.754242919199539 ], "confidence_level": 0.95, "mean": 1.724660737043224, "replicates": 12, "sample_sd": 0.05199986270978758, "standard_error": 0.015011067366659721 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1981891666184772, 0.20204015543856496 ], "confidence_level": 0.95, "mean": 0.19992742603064345, "replicates": 12, "sample_sd": 0.0035618422896455063, "standard_error": 0.0010282153023689132 }, "log_loss": { "bootstrap_interval": [ 0.5797929156902167, 0.5884856389284587 ], "confidence_level": 0.95, "mean": 0.5838355487280672, "replicates": 12, "sample_sd": 0.007996384601348563, "standard_error": 0.002308357401066186 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18951787263017328, 0.19326846391339986 ], "confidence_level": 0.95, "mean": 0.19108265480962047, "replicates": 12, "sample_sd": 0.003470697465198434, "standard_error": 0.0010019040579040339 }, "log_loss": { "bootstrap_interval": [ 0.5592556394558061, 0.5675286215255241 ], "confidence_level": 0.95, "mean": 0.5628580402783209, "replicates": 12, "sample_sd": 0.007747423484770867, "standard_error": 0.0022364885172292447 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1111111111111111, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.12195121951219512, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 1000 } }, "config": { "conditions": [ { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 } ], "phase": "exploratory-stress-groups-m1000-v1", "replicates": 12, "schema_version": 1, "seed": 84173102, "test_count": 2000, "train_count": 1000 }, "config_sha256": "64419cb9db87931de287f1a9e9946d0da4f75a965c21b6c4d4950778d8599aa8", "expected_runs": 144, "input_path": "results/stress-v1/stress-groups-m1000-v1.json", "input_sha256": "924641025e4c7eb25bfb36c04e03a56d3d0f4294c81fc154858e9b09d7164a1b", "ledger_receipt": { "bytes": 108900874, "count": 432000, "payload_sha256": "2fc87c20da86cbccecaaa8647d7398ebcbccf10ab53be25eefc4e88f72ceec4e", "schema_version": 1, "sha256": "c700f96d00e4d75794fb7bff0b35a79f1323fd882cd8fad175a5d7f70ce87b70" }, "manifest_sha256": null, "observed_runs": 144, "phase": "exploratory-stress-groups-m1000-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" }, "exploratory-stress-groups-m200-v1": { "analysis_seed": 5385974971768605997056808436048018736376208838761438107644507165283579182961, "cells": { "groups-4-3-2-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12203785942452816, -0.11952138096094878 ], "confidence_level": 0.95, "mean": -0.1208497590301434, "replicates": 12, "sample_sd": 0.0023718029452810914, "standard_error": 0.0006846805344613927 }, "log_loss": { "bootstrap_interval": [ -0.3737143574881583, -0.36551453941106515 ], "confidence_level": 0.95, "mean": -0.36980508076462293, "replicates": 12, "sample_sd": 0.007633005661198534, "standard_error": 0.0022034589366094555 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 9.335996454562729e-06, 0.0003340445938808216 ], "confidence_level": 0.95, "mean": 0.00014584370965250483, "replicates": 12, "sample_sd": 0.0003024697008993017, "standard_error": 8.731548161795872e-05 }, "log_loss": { "bootstrap_interval": [ 0.0001009506407853508, 0.0027200008631128383 ], "confidence_level": 0.95, "mean": 0.001154721635959426, "replicates": 12, "sample_sd": 0.0026161911154972428, "standard_error": 0.0007552293223919203 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 9.335996454562165e-06, 0.0003340445938808222 ], "confidence_level": 0.95, "mean": 0.00014584370965250483, "replicates": 12, "sample_sd": 0.0003024697008993026, "standard_error": 8.731548161795897e-05 }, "log_loss": { "bootstrap_interval": [ 0.00010095064078535312, 0.0027200008631128383 ], "confidence_level": 0.95, "mean": 0.0011547216359594306, "replicates": 12, "sample_sd": 0.002616191115497242, "standard_error": 0.00075522932239192 } } }, "discount": { "bootstrap_interval": [ 0.8998812002678168, 0.985693681942416 ], "confidence_level": 0.95, "mean": 0.9461240561831215, "replicates": 12, "sample_sd": 0.07863795467940007, "standard_error": 0.022700822151336612 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7651424940530456, "discount_one_count": 7, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16188621672839243, 0.162834082860089 ], "confidence_level": 0.95, "mean": 0.16236605327424372, "replicates": 12, "sample_sd": 0.0008752554684856174, "standard_error": 0.0002526644901699316 }, "log_loss": { "bootstrap_interval": [ 0.5120550888419727, 0.5140454304064859 ], "confidence_level": 0.95, "mean": 0.5130507765650558, "replicates": 12, "sample_sd": 0.001849287610909243, "standard_error": 0.0005338433499837456 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.03984460414569452, 0.042987210297430495 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.002919907694614284, "standard_error": 0.0008429047467472082 }, "log_loss": { "bootstrap_interval": [ 0.13706880802576563, 0.14735278621401782 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.039894826427478096, 0.04302572746544945 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.0029199076946142856, "standard_error": 0.0008429047467472088 }, "log_loss": { "bootstrap_interval": [ 0.13715820828340905, 0.14733446136378925 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "independent": { "brier": { "bootstrap_interval": [ 0.0398483979160728, 0.04303377027252327 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.002919907694614284, "standard_error": 0.0008429047467472082 }, "log_loss": { "bootstrap_interval": [ 0.13717096269483872, 0.14740601478699447 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1947815946678808, 0.20074081402977834 ], "confidence_level": 0.95, "mean": 0.19798151872987868, "replicates": 12, "sample_sd": 0.005536209637648682, "standard_error": 0.0015981660622933337 }, "log_loss": { "bootstrap_interval": [ 0.5720579244783963, 0.5863647828216407 ], "confidence_level": 0.95, "mean": 0.5798492083256285, "replicates": 12, "sample_sd": 0.013175692779153196, "standard_error": 0.0038034948864019533 } }, "learned": { "brier": { "bootstrap_interval": [ 0.04003338234167634, 0.04315436861209269 ], "confidence_level": 0.95, "mean": 0.041516294244100325, "replicates": 12, "sample_sd": 0.0028432879231083713, "standard_error": 0.0008207865238951151 }, "log_loss": { "bootstrap_interval": [ 0.13871772371226437, 0.1480142066770908 ], "confidence_level": 0.95, "mean": 0.14324569580043295, "replicates": 12, "sample_sd": 0.008633876235227687, "standard_error": 0.002492385384279309 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0, 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-4-3-2-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07274844169779991, -0.06976225568614501 ], "confidence_level": 0.95, "mean": -0.07128438523430601, "replicates": 12, "sample_sd": 0.0027873871391791505, "standard_error": 0.0008046493575703918 }, "log_loss": { "bootstrap_interval": [ -0.21877485981629588, -0.20769871902963027 ], "confidence_level": 0.95, "mean": -0.21317723978319056, "replicates": 12, "sample_sd": 0.010304495174729303, "standard_error": 0.0029746515314965816 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.006333828393146886, 0.008352740995159825 ], "confidence_level": 0.95, "mean": 0.007381736039327739, "replicates": 12, "sample_sd": 0.0018568251451063409, "standard_error": 0.000536019248682606 }, "log_loss": { "bootstrap_interval": [ 0.018608009777109146, 0.02536961649764335 ], "confidence_level": 0.95, "mean": 0.021919258124789533, "replicates": 12, "sample_sd": 0.0062459106200900595, "standard_error": 0.0018030390889216693 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.007698736520470907, -0.006619625103104645 ], "confidence_level": 0.95, "mean": -0.00714482194076172, "replicates": 12, "sample_sd": 0.0010036938627147815, "standard_error": 0.0002897414609111772 }, "log_loss": { "bootstrap_interval": [ -0.07502039786057649, -0.06271577193782726 ], "confidence_level": 0.95, "mean": -0.06877030601270157, "replicates": 12, "sample_sd": 0.011421516558120348, "standard_error": 0.0032971078296922755 } } }, "discount": { "bootstrap_interval": [ 0.5099503496393869, 0.5754038793739802 ], "confidence_level": 0.95, "mean": 0.5408787509826114, "replicates": 12, "sample_sd": 0.06030560367532527, "standard_error": 0.017408728257795963 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.67164635448276, "discount_min": 0.45063071315013203, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1653957064956068, 0.166980019350418 ], "confidence_level": 0.95, "mean": 0.16618885865474273, "replicates": 12, "sample_sd": 0.0014545813117582268, "standard_error": 0.00041990145595090564 }, "log_loss": { "bootstrap_interval": [ 0.5169497445110617, 0.5202336400389354 ], "confidence_level": 0.95, "mean": 0.518637318913596, "replicates": 12, "sample_sd": 0.0030707681454884743, "standard_error": 0.0008864544077083493 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.11941065740356807, 0.12189423899647239 ], "confidence_level": 0.95, "mean": 0.12064067158365907, "replicates": 12, "sample_sd": 0.0023297247269378437, "standard_error": 0.0006725335991176458 }, "log_loss": { "bootstrap_interval": [ 0.3995657110096796, 0.4054301654360789 ], "confidence_level": 0.95, "mean": 0.402525149949211, "replicates": 12, "sample_sd": 0.005424330822286911, "standard_error": 0.001565869430210466 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.08538722084642403, 0.08960037621432217 ], "confidence_level": 0.95, "mean": 0.08752273738110897, "replicates": 12, "sample_sd": 0.003901349424080768, "standard_error": 0.0011262225700979114 }, "log_loss": { "bootstrap_interval": [ 0.2769203946597286, 0.29010499768906217 ], "confidence_level": 0.95, "mean": 0.28354082100561584, "replicates": 12, "sample_sd": 0.012344743842943933, "standard_error": 0.003563620590400328 } }, "independent": { "brier": { "bootstrap_interval": [ 0.09967433262171965, 0.1045818267619016 ], "confidence_level": 0.95, "mean": 0.10204929536119843, "replicates": 12, "sample_sd": 0.0045759457161978845, "standard_error": 0.0013209617455219819 }, "log_loss": { "bootstrap_interval": [ 0.3621890539495109, 0.38672843911155763 ], "confidence_level": 0.95, "mean": 0.37423038514310697, "replicates": 12, "sample_sd": 0.022745913864038524, "standard_error": 0.006566179746183342 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1960530972698211, 0.200045565142493 ], "confidence_level": 0.95, "mean": 0.198204636536768, "replicates": 12, "sample_sd": 0.003599045641874691, "standard_error": 0.0010389549850810512 }, "log_loss": { "bootstrap_interval": [ 0.5748716412776318, 0.5846336513918791 ], "confidence_level": 0.95, "mean": 0.5800455524940334, "replicates": 12, "sample_sd": 0.00908889936602059, "standard_error": 0.0026237392478047033 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09273025382416737, 0.0970513002462791 ], "confidence_level": 0.95, "mean": 0.0949044734204367, "replicates": 12, "sample_sd": 0.0039770491539326365, "standard_error": 0.0011480751998016905 }, "log_loss": { "bootstrap_interval": [ 0.2986116427238965, 0.3122968751410025 ], "confidence_level": 0.95, "mean": 0.3054600791304054, "replicates": 12, "sample_sd": 0.012990427473472947, "standard_error": 0.0037500133993489584 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.4, 0.5, 0.6666666666666666, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-4-3-2-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04928295725219473, -0.04664849715295806 ], "confidence_level": 0.95, "mean": -0.048001729638239815, "replicates": 12, "sample_sd": 0.0024603852479811094, "standard_error": 0.0007102520426160389 }, "log_loss": { "bootstrap_interval": [ -0.14594174909183177, -0.13670781829326203 ], "confidence_level": 0.95, "mean": -0.14150885294035487, "replicates": 12, "sample_sd": 0.008617929648483203, "standard_error": 0.002487782001204517 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.011802250075541264, 0.015061288596201853 ], "confidence_level": 0.95, "mean": 0.013440820568677368, "replicates": 12, "sample_sd": 0.002982944308004675, "standard_error": 0.0008611018496020806 }, "log_loss": { "bootstrap_interval": [ 0.03323052930827977, 0.042912529028104704 ], "confidence_level": 0.95, "mean": 0.037892819257924, "replicates": 12, "sample_sd": 0.00884016525028144, "standard_error": 0.002551935893465383 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.017063301410274834, -0.01509788932897672 ], "confidence_level": 0.95, "mean": -0.016127884216983836, "replicates": 12, "sample_sd": 0.0018529662455549984, "standard_error": 0.0005349052803352343 }, "log_loss": { "bootstrap_interval": [ -0.19907568306968634, -0.17679592802839558 ], "confidence_level": 0.95, "mean": -0.18781269652034652, "replicates": 12, "sample_sd": 0.02066474882623825, "standard_error": 0.005965399148782329 } } }, "discount": { "bootstrap_interval": [ 0.3505002479607844, 0.39590588784148295 ], "confidence_level": 0.95, "mean": 0.37192341840173465, "replicates": 12, "sample_sd": 0.04281181065258189, "standard_error": 0.012358705202381722 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.4507359320177725, "discount_min": 0.31476123677686585, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16836056627466844, 0.17064942698933627 ], "confidence_level": 0.95, "mean": 0.1694772259446456, "replicates": 12, "sample_sd": 0.002112417153750353, "standard_error": 0.0006098023061792746 }, "log_loss": { "bootstrap_interval": [ 0.5214469635074689, 0.5263236909168224 ], "confidence_level": 0.95, "mean": 0.5238014203394681, "replicates": 12, "sample_sd": 0.004527199194717653, "standard_error": 0.001306889836872647 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.15537300324499148, 0.15796034253284028 ], "confidence_level": 0.95, "mean": 0.15662577292449026, "replicates": 12, "sample_sd": 0.0023827864185993475, "standard_error": 0.0006878511900998588 }, "log_loss": { "bootstrap_interval": [ 0.49006731441425305, 0.4957755045803825 ], "confidence_level": 0.95, "mean": 0.492844522193319, "replicates": 12, "sample_sd": 0.0052597918782521695, "standard_error": 0.0015183711283951488 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.10598384572056056, 0.10991133382483881 ], "confidence_level": 0.95, "mean": 0.1080346757377284, "replicates": 12, "sample_sd": 0.0037057348578539645, "standard_error": 0.001069753508863683 }, "log_loss": { "bootstrap_interval": [ 0.3383883268440012, 0.3500622647563424 ], "confidence_level": 0.95, "mean": 0.3443997481411892, "replicates": 12, "sample_sd": 0.010851589868144313, "standard_error": 0.003132584165754268 } }, "independent": { "brier": { "bootstrap_interval": [ 0.13446405379642629, 0.1407076815014275 ], "confidence_level": 0.95, "mean": 0.1376033805233896, "replicates": 12, "sample_sd": 0.005768577117521727, "standard_error": 0.0016652447758211423 }, "log_loss": { "bootstrap_interval": [ 0.5536658890065494, 0.5875349941350289 ], "confidence_level": 0.95, "mean": 0.5701052639194597, "replicates": 12, "sample_sd": 0.031703516680794484, "standard_error": 0.009152016944957243 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1959750461911333, 0.2007041175394299 ], "confidence_level": 0.95, "mean": 0.19838127447040257, "replicates": 12, "sample_sd": 0.004438570010571056, "standard_error": 0.0012813047952100998 }, "log_loss": { "bootstrap_interval": [ 0.5739606629621643, 0.5859718851283873 ], "confidence_level": 0.95, "mean": 0.5802357830613853, "replicates": 12, "sample_sd": 0.011006438851788843, "standard_error": 0.0031772852169497223 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11923329541508844, 0.12379816692132567 ], "confidence_level": 0.95, "mean": 0.12147549630640578, "replicates": 12, "sample_sd": 0.0043484860061167325, "standard_error": 0.001255299783099408 }, "log_loss": { "bootstrap_interval": [ 0.37566085548296124, 0.3894644992928265 ], "confidence_level": 0.95, "mean": 0.38229256739911316, "replicates": 12, "sample_sd": 0.012816789618729361, "standard_error": 0.0036998884682600992 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.27027027027027023, 0.35714285714285715, 0.5263157894736842, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.35714285714285715, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-4-3-2-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.044871276568132626, -0.042221491517294 ], "confidence_level": 0.95, "mean": -0.04352858920104486, "replicates": 12, "sample_sd": 0.0024914143176406637, "standard_error": 0.0007192093634763626 }, "log_loss": { "bootstrap_interval": [ -0.13253197883485263, -0.12260531694692962 ], "confidence_level": 0.95, "mean": -0.12761503813920994, "replicates": 12, "sample_sd": 0.008993087514901651, "standard_error": 0.0025960807487871657 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.012835499912477446, 0.016344600266340645 ], "confidence_level": 0.95, "mean": 0.014599484615683453, "replicates": 12, "sample_sd": 0.0032151004756079573, "standard_error": 0.0009281195625319741 }, "log_loss": { "bootstrap_interval": [ 0.03604747749557521, 0.04578998696567002 ], "confidence_level": 0.95, "mean": 0.04094489239046571, "replicates": 12, "sample_sd": 0.009073508915940193, "standard_error": 0.00261929640755627 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.019946002016014503, -0.01727258419504187 ], "confidence_level": 0.95, "mean": -0.018646485900367545, "replicates": 12, "sample_sd": 0.0025174005216465328, "standard_error": 0.0007267109344153651 }, "log_loss": { "bootstrap_interval": [ -0.2345435721376596, -0.20980334981161025 ], "confidence_level": 0.95, "mean": -0.22229513484772806, "replicates": 12, "sample_sd": 0.02262301079577765, "standard_error": 0.006530700686411019 } } }, "discount": { "bootstrap_interval": [ 0.3156503823595245, 0.3591162997118404 ], "confidence_level": 0.95, "mean": 0.33773865271229936, "replicates": 12, "sample_sd": 0.04006231257649123, "standard_error": 0.011564993475198073 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3989223018514785, "discount_min": 0.27427833304661875, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1691354659797457, 0.17161544451666558 ], "confidence_level": 0.95, "mean": 0.17034306653473677, "replicates": 12, "sample_sd": 0.002326684278484401, "standard_error": 0.0006716558972511197 }, "log_loss": { "bootstrap_interval": [ 0.5225998168938796, 0.5279688639205796 ], "confidence_level": 0.95, "mean": 0.5252248580338683, "replicates": 12, "sample_sd": 0.005005119306380595, "standard_error": 0.0014448534894325148 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.16160626350205162, 0.16428528402248796 ], "confidence_level": 0.95, "mean": 0.1628996241564258, "replicates": 12, "sample_sd": 0.002469513109039044, "standard_error": 0.0007128870291355008 }, "log_loss": { "bootstrap_interval": [ 0.5047038297974201, 0.5105797470055393 ], "confidence_level": 0.95, "mean": 0.5075709897499605, "replicates": 12, "sample_sd": 0.005402096381621098, "standard_error": 0.0015594509000586222 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.11027879249678076, 0.11387833507139807 ], "confidence_level": 0.95, "mean": 0.11221499271800844, "replicates": 12, "sample_sd": 0.0033229303408310175, "standard_error": 0.0009592473633885815 }, "log_loss": { "bootstrap_interval": [ 0.3514815124122784, 0.36162545622379727 ], "confidence_level": 0.95, "mean": 0.3566649275041927, "replicates": 12, "sample_sd": 0.009481180888297393, "standard_error": 0.0027369811690470176 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1418874465288769, 0.14895589247408553 ], "confidence_level": 0.95, "mean": 0.14546096323405944, "replicates": 12, "sample_sd": 0.006523140040196889, "standard_error": 0.0018830683290846503 }, "log_loss": { "bootstrap_interval": [ 0.6016374549804058, 0.6392497999156534 ], "confidence_level": 0.95, "mean": 0.6199049547423865, "replicates": 12, "sample_sd": 0.03489284067020404, "standard_error": 0.010072695476866513 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19549809749880057, 0.20114091002892254 ], "confidence_level": 0.95, "mean": 0.19836684545746547, "replicates": 12, "sample_sd": 0.005192545167919238, "standard_error": 0.0014989586752387314 }, "log_loss": { "bootstrap_interval": [ 0.5729940332350408, 0.5868319479108904 ], "confidence_level": 0.95, "mean": 0.5801052504343464, "replicates": 12, "sample_sd": 0.012584439671942404, "standard_error": 0.0036328148160982765 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12441820360379507, 0.1293130909844796 ], "confidence_level": 0.95, "mean": 0.1268144773336919, "replicates": 12, "sample_sd": 0.004531988392991595, "standard_error": 0.0013082723593289785 }, "log_loss": { "bootstrap_interval": [ 0.3904780896565078, 0.40535660107030363 ], "confidence_level": 0.95, "mean": 0.3976098198946584, "replicates": 12, "sample_sd": 0.013449489094655844, "standard_error": 0.003882533074631244 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25, 0.3333333333333333, 0.5, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-5-5-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12205983118256303, -0.11952157492737872 ], "confidence_level": 0.95, "mean": -0.1208497590301434, "replicates": 12, "sample_sd": 0.0023718029452810914, "standard_error": 0.0006846805344613927 }, "log_loss": { "bootstrap_interval": [ -0.37373320439185226, -0.36560416891439534 ], "confidence_level": 0.95, "mean": -0.36980508076462293, "replicates": 12, "sample_sd": 0.007633005661198534, "standard_error": 0.0022034589366094555 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 9.335996454562165e-06, 0.00032858509252165435 ], "confidence_level": 0.95, "mean": 0.00014584370965250483, "replicates": 12, "sample_sd": 0.0003024697008993017, "standard_error": 8.731548161795872e-05 }, "log_loss": { "bootstrap_interval": [ 0.0001009506407853485, 0.0027723197942758535 ], "confidence_level": 0.95, "mean": 0.001154721635959426, "replicates": 12, "sample_sd": 0.0026161911154972428, "standard_error": 0.0007552293223919203 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 9.335996454562165e-06, 0.000328585092521655 ], "confidence_level": 0.95, "mean": 0.00014584370965250483, "replicates": 12, "sample_sd": 0.0003024697008993026, "standard_error": 8.731548161795897e-05 }, "log_loss": { "bootstrap_interval": [ 0.00010095064078535312, 0.0027723197942758648 ], "confidence_level": 0.95, "mean": 0.0011547216359594306, "replicates": 12, "sample_sd": 0.002616191115497242, "standard_error": 0.00075522932239192 } } }, "discount": { "bootstrap_interval": [ 0.8996617416557771, 0.9856211723088635 ], "confidence_level": 0.95, "mean": 0.9461240561831215, "replicates": 12, "sample_sd": 0.07863795467940007, "standard_error": 0.022700822151336612 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7651424940530456, "discount_one_count": 7, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16190890725375814, 0.16283120656080197 ], "confidence_level": 0.95, "mean": 0.16236605327424372, "replicates": 12, "sample_sd": 0.0008752554684856174, "standard_error": 0.0002526644901699316 }, "log_loss": { "bootstrap_interval": [ 0.5120621209187468, 0.5140367345637268 ], "confidence_level": 0.95, "mean": 0.5130507765650558, "replicates": 12, "sample_sd": 0.001849287610909243, "standard_error": 0.0005338433499837456 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.03986309584045603, 0.04301587308137093 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.002919907694614284, "standard_error": 0.0008429047467472082 }, "log_loss": { "bootstrap_interval": [ 0.13713730863468623, 0.14737282359189352 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.03983249177675164, 0.043010138481899954 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.0029199076946142856, "standard_error": 0.0008429047467472088 }, "log_loss": { "bootstrap_interval": [ 0.13717839664896794, 0.14742369263297664 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "independent": { "brier": { "bootstrap_interval": [ 0.03985040863672934, 0.04298415136225754 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.002919907694614284, "standard_error": 0.0008429047467472082 }, "log_loss": { "bootstrap_interval": [ 0.13717262035133476, 0.14731297589404244 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19481340839527364, 0.20072193973894045 ], "confidence_level": 0.95, "mean": 0.19798151872987868, "replicates": 12, "sample_sd": 0.005536209637648682, "standard_error": 0.0015981660622933337 }, "log_loss": { "bootstrap_interval": [ 0.5724067748388262, 0.5863934972135473 ], "confidence_level": 0.95, "mean": 0.5798492083256285, "replicates": 12, "sample_sd": 0.013175692779153196, "standard_error": 0.0038034948864019533 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0400266411572941, 0.043110940745882176 ], "confidence_level": 0.95, "mean": 0.041516294244100325, "replicates": 12, "sample_sd": 0.0028432879231083713, "standard_error": 0.0008207865238951151 }, "log_loss": { "bootstrap_interval": [ 0.1386056603643515, 0.1479789340237791 ], "confidence_level": 0.95, "mean": 0.14324569580043295, "replicates": 12, "sample_sd": 0.008633876235227687, "standard_error": 0.002492385384279309 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-5-5-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04492931147850793, -0.04193416402698891 ], "confidence_level": 0.95, "mean": -0.043563061584850134, "replicates": 12, "sample_sd": 0.0027636212913752884, "standard_error": 0.000797788748256852 }, "log_loss": { "bootstrap_interval": [ -0.13311344050863838, -0.12407753433349412 ], "confidence_level": 0.95, "mean": -0.12901325500157138, "replicates": 12, "sample_sd": 0.008360354706631335, "standard_error": 0.002413426520197178 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 9.175928076660493e-05, 0.00045866746689274853 ], "confidence_level": 0.95, "mean": 0.0002690652055219028, "replicates": 12, "sample_sd": 0.00034207731910273863, "standard_error": 9.874921613381584e-05 }, "log_loss": { "bootstrap_interval": [ 0.0004993749256326366, 0.002325917042447718 ], "confidence_level": 0.95, "mean": 0.0013633280815371952, "replicates": 12, "sample_sd": 0.001678492150981774, "standard_error": 0.0004845389476010006 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.019847956045069594, -0.017460918570297225 ], "confidence_level": 0.95, "mean": -0.01864223699809796, "replicates": 12, "sample_sd": 0.00223327156082582, "standard_error": 0.0006446899684081614 }, "log_loss": { "bootstrap_interval": [ -0.2299812647139851, -0.20534614529891992 ], "confidence_level": 0.95, "mean": -0.2171793585001058, "replicates": 12, "sample_sd": 0.02250604391712606, "standard_error": 0.006496935256973135 } } }, "discount": { "bootstrap_interval": [ 0.3345014910095901, 0.3720342952582017 ], "confidence_level": 0.95, "mean": 0.3528424835799354, "replicates": 12, "sample_sd": 0.03469514459671751, "standard_error": 0.010015625536243923 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.4127102959357013, "discount_min": 0.2901476606360639, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16898118709772791, 0.17119907265463638 ], "confidence_level": 0.95, "mean": 0.1700718321046038, "replicates": 12, "sample_sd": 0.002068599417938059, "standard_error": 0.0005971532153960208 }, "log_loss": { "bootstrap_interval": [ 0.5223381174764568, 0.5271199381799855 ], "confidence_level": 0.95, "mean": 0.5246446045154789, "replicates": 12, "sample_sd": 0.004367123209894492, "standard_error": 0.001260679880408424 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1344469341427008, 0.13809373798926547 ], "confidence_level": 0.95, "mean": 0.13620344643251725, "replicates": 12, "sample_sd": 0.0033851700884019002, "standard_error": 0.0009772144308957532 }, "log_loss": { "bootstrap_interval": [ 0.42885840448903007, 0.43722702251476936 ], "confidence_level": 0.95, "mean": 0.4329236747849337, "replicates": 12, "sample_sd": 0.007813742861839126, "standard_error": 0.0022556332723306684 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.1238834377708874, 0.12875231253301514 ], "confidence_level": 0.95, "mean": 0.12623970531423176, "replicates": 12, "sample_sd": 0.004537871265485336, "standard_error": 0.0013099705983379132 }, "log_loss": { "bootstrap_interval": [ 0.3879722319629151, 0.40087920569211244 ], "confidence_level": 0.95, "mean": 0.39426802143237033, "replicates": 12, "sample_sd": 0.012004222342785285, "standard_error": 0.003465320500509602 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14183685481897013, 0.14887674476552779 ], "confidence_level": 0.95, "mean": 0.14515100751785162, "replicates": 12, "sample_sd": 0.006520268940453002, "standard_error": 0.0018822395139796485 }, "log_loss": { "bootstrap_interval": [ 0.5950605416935255, 0.6313556570422912 ], "confidence_level": 0.95, "mean": 0.6128107080140134, "replicates": 12, "sample_sd": 0.033266629309399325, "standard_error": 0.009603248693406598 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19612901487682327, 0.2000626337557619 ], "confidence_level": 0.95, "mean": 0.198204636536768, "replicates": 12, "sample_sd": 0.003599045641874691, "standard_error": 0.0010389549850810512 }, "log_loss": { "bootstrap_interval": [ 0.5749643516123758, 0.5846940320053787 ], "confidence_level": 0.95, "mean": 0.5800455524940334, "replicates": 12, "sample_sd": 0.00908889936602059, "standard_error": 0.0026237392478047033 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1241712509963801, 0.12907040186531954 ], "confidence_level": 0.95, "mean": 0.12650877051975368, "replicates": 12, "sample_sd": 0.004578613246767624, "standard_error": 0.0013217317952682371 }, "log_loss": { "bootstrap_interval": [ 0.3894427880486006, 0.40265812926891037 ], "confidence_level": 0.95, "mean": 0.39563134951390755, "replicates": 12, "sample_sd": 0.012208551366783352, "standard_error": 0.003524305209013871 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333, 0.3333333333333333 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-5-5-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.020647924427358175, -0.017747126874289514 ], "confidence_level": 0.95, "mean": -0.019261021074542622, "replicates": 12, "sample_sd": 0.002664518059040169, "standard_error": 0.0007691801093237304 }, "log_loss": { "bootstrap_interval": [ -0.060615133231606506, -0.05199896272947651 ], "confidence_level": 0.95, "mean": -0.05651704642362191, "replicates": 12, "sample_sd": 0.008078549130348923, "standard_error": 0.002332076257534284 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.00025921887781689906, 0.0011040791702480637 ], "confidence_level": 0.95, "mean": 0.0006553442440107882, "replicates": 12, "sample_sd": 0.000789660008038319, "standard_error": 0.00022795520910460278 }, "log_loss": { "bootstrap_interval": [ 0.0010411148634055102, 0.00450495843466652 ], "confidence_level": 0.95, "mean": 0.0026670683119585695, "replicates": 12, "sample_sd": 0.0032253661676313753, "standard_error": 0.0009310830125585432 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0388963322880507, -0.03443664213945417 ], "confidence_level": 0.95, "mean": -0.03658522639972258, "replicates": 12, "sample_sd": 0.00412901634150483, "standard_error": 0.0011919443481280888 }, "log_loss": { "bootstrap_interval": [ -0.5255379658566094, -0.48427617823064206 ], "confidence_level": 0.95, "mean": -0.5045750488755939, "replicates": 12, "sample_sd": 0.03824222296117293, "standard_error": 0.01103957886052144 } } }, "discount": { "bootstrap_interval": [ 0.2088683189457809, 0.24453171096248888 ], "confidence_level": 0.95, "mean": 0.22706812626961684, "replicates": 12, "sample_sd": 0.033115162866437974, "standard_error": 0.009559524097598132 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.28279403924817026, "discount_min": 0.16975519105602682, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1750359183064213, 0.17840552072517213 ], "confidence_level": 0.95, "mean": 0.1766325779824803, "replicates": 12, "sample_sd": 0.003108561003090951, "standard_error": 0.0008973642659634668 }, "log_loss": { "bootstrap_interval": [ 0.5322620015899122, 0.5395279495845499 ], "confidence_level": 0.95, "mean": 0.5356977024911405, "replicates": 12, "sample_sd": 0.0066221887838290965, "standard_error": 0.0019116612384841249 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1670584807309995, 0.17091829935023764 ], "confidence_level": 0.95, "mean": 0.16894609037822034, "replicates": 12, "sample_sd": 0.0035675229977669134, "standard_error": 0.0010298551815504542 }, "log_loss": { "bootstrap_interval": [ 0.5117355911526886, 0.5202031083041777 ], "confidence_level": 0.95, "mean": 0.5157883666820358, "replicates": 12, "sample_sd": 0.0077308531154395765, "standard_error": 0.0022317050636322484 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15389422910338077, 0.15969839695197385 ], "confidence_level": 0.95, "mean": 0.15671621266392688, "replicates": 12, "sample_sd": 0.005381171052505914, "standard_error": 0.0015534102778598559 }, "log_loss": { "bootstrap_interval": [ 0.46981343134738957, 0.483552304369021 ], "confidence_level": 0.95, "mean": 0.47651358775555996, "replicates": 12, "sample_sd": 0.013018089184223798, "standard_error": 0.003757998647423083 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1892122338522341, 0.19920316548077324 ], "confidence_level": 0.95, "mean": 0.19395678330766025, "replicates": 12, "sample_sd": 0.00922655414011242, "standard_error": 0.002663476758243281 }, "log_loss": { "bootstrap_interval": [ 0.9574427698243043, 1.0120606282025986 ], "confidence_level": 0.95, "mean": 0.9837557049431124, "replicates": 12, "sample_sd": 0.05077737916733105, "standard_error": 0.014658166765501138 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19588611400269706, 0.20072762272189043 ], "confidence_level": 0.95, "mean": 0.19838127447040257, "replicates": 12, "sample_sd": 0.004438570010571056, "standard_error": 0.0012813047952100998 }, "log_loss": { "bootstrap_interval": [ 0.5741075165173495, 0.585841709810596 ], "confidence_level": 0.95, "mean": 0.5802357830613853, "replicates": 12, "sample_sd": 0.011006438851788843, "standard_error": 0.0031772852169497223 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15450189566166636, 0.16039714157924517 ], "confidence_level": 0.95, "mean": 0.15737155690793767, "replicates": 12, "sample_sd": 0.005486679182210697, "standard_error": 0.001583867851403231 }, "log_loss": { "bootstrap_interval": [ 0.4719942364239076, 0.48671978099915814 ], "confidence_level": 0.95, "mean": 0.4791806560675185, "replicates": 12, "sample_sd": 0.013948245818622838, "standard_error": 0.004026511739052484 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2173913043478261, 0.2173913043478261 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-5-5-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.016878144129830047, -0.013887637651073488 ], "confidence_level": 0.95, "mean": -0.015408300458321532, "replicates": 12, "sample_sd": 0.002753314409696821, "standard_error": 0.0007948134078010677 }, "log_loss": { "bootstrap_interval": [ -0.04938281427353744, -0.039853690135783014 ], "confidence_level": 0.95, "mean": -0.044836796889943596, "replicates": 12, "sample_sd": 0.008678791806525098, "standard_error": 0.002505351392868992 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.0002959422079525588, 0.0014270307389735643 ], "confidence_level": 0.95, "mean": 0.0008371640570617548, "replicates": 12, "sample_sd": 0.0010580121594620578, "standard_error": 0.0003054218025356582 }, "log_loss": { "bootstrap_interval": [ 0.0011584466390493692, 0.005696084466234739 ], "confidence_level": 0.95, "mean": 0.003273633239712396, "replicates": 12, "sample_sd": 0.0041426112274895064, "standard_error": 0.0011958688536695163 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04221577735203086, -0.03728187336951784 ], "confidence_level": 0.95, "mean": -0.03966986613460218, "replicates": 12, "sample_sd": 0.004628213288971731, "standard_error": 0.0013360500941274162 }, "log_loss": { "bootstrap_interval": [ -0.6030856081555164, -0.5572647183522418 ], "confidence_level": 0.95, "mean": -0.5796603555691525, "replicates": 12, "sample_sd": 0.042153513418100905, "standard_error": 0.01216867115961453 } } }, "discount": { "bootstrap_interval": [ 0.18873345450815798, 0.2246258379410577 ], "confidence_level": 0.95, "mean": 0.2069420557326549, "replicates": 12, "sample_sd": 0.03351156202490401, "standard_error": 0.009673954678021585 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.2582848482881005, "discount_min": 0.15097463243113085, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17656121139916392, 0.18031376542408525 ], "confidence_level": 0.95, "mean": 0.17834578475334303, "replicates": 12, "sample_sd": 0.0034187603555398583, "standard_error": 0.000986911105782879 }, "log_loss": { "bootstrap_interval": [ 0.5349922356476811, 0.5429634128274619 ], "confidence_level": 0.95, "mean": 0.5387736544644552, "replicates": 12, "sample_sd": 0.007300626341084495, "standard_error": 0.002107509291639003 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.17256705665366973, 0.17653664241216463 ], "confidence_level": 0.95, "mean": 0.17446531787663097, "replicates": 12, "sample_sd": 0.0036440766873998436, "standard_error": 0.00105195432820897 }, "log_loss": { "bootstrap_interval": [ 0.5249125433405114, 0.5333745077365617 ], "confidence_level": 0.95, "mean": 0.5290407651706844, "replicates": 12, "sample_sd": 0.007839903478102867, "standard_error": 0.002263185191751687 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15920102668958797, 0.16529838898146446 ], "confidence_level": 0.95, "mean": 0.16210032023795976, "replicates": 12, "sample_sd": 0.005645383574176982, "standard_error": 0.0016296818631148862 }, "log_loss": { "bootstrap_interval": [ 0.48379732142665993, 0.49815497682843274 ], "confidence_level": 0.95, "mean": 0.4906632243347993, "replicates": 12, "sample_sd": 0.013520279358365025, "standard_error": 0.003902968463535494 } }, "independent": { "brier": { "bootstrap_interval": [ 0.19755757303359117, 0.20804692684454337 ], "confidence_level": 0.95, "mean": 0.2026073504296237, "replicates": 12, "sample_sd": 0.009871034681829259, "standard_error": 0.0028495222653671273 }, "log_loss": { "bootstrap_interval": [ 1.0437632408184838, 1.104309546967158 ], "confidence_level": 0.95, "mean": 1.0735972131436642, "replicates": 12, "sample_sd": 0.0560338348760514, "standard_error": 0.01617557482470766 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19560824710620278, 0.20116868066918872 ], "confidence_level": 0.95, "mean": 0.19836684545746547, "replicates": 12, "sample_sd": 0.005192545167919238, "standard_error": 0.0014989586752387314 }, "log_loss": { "bootstrap_interval": [ 0.5730585980208366, 0.5867588813067947 ], "confidence_level": 0.95, "mean": 0.5801052504343464, "replicates": 12, "sample_sd": 0.012584439671942404, "standard_error": 0.0036328148160982765 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15985551209822554, 0.1660885847234912 ], "confidence_level": 0.95, "mean": 0.1629374842950215, "replicates": 12, "sample_sd": 0.005777659035300507, "standard_error": 0.0016678664996583109 }, "log_loss": { "bootstrap_interval": [ 0.48627679297617427, 0.5020359922972005 ], "confidence_level": 0.95, "mean": 0.49393685757451167, "replicates": 12, "sample_sd": 0.014926231717054856, "standard_error": 0.004308831949914176 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 0.2 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-9-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12206041626230042, -0.11952798709332865 ], "confidence_level": 0.95, "mean": -0.1208497590301434, "replicates": 12, "sample_sd": 0.0023718029452810914, "standard_error": 0.0006846805344613927 }, "log_loss": { "bootstrap_interval": [ -0.3737243813276303, -0.3655984479982514 ], "confidence_level": 0.95, "mean": -0.36980508076462293, "replicates": 12, "sample_sd": 0.007633005661198534, "standard_error": 0.0022034589366094555 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 8.673794285281759e-06, 0.00032856853746742257 ], "confidence_level": 0.95, "mean": 0.00014584370965250483, "replicates": 12, "sample_sd": 0.0003024697008993017, "standard_error": 8.731548161795872e-05 }, "log_loss": { "bootstrap_interval": [ 0.00010095064078535312, 0.002806681087601177 ], "confidence_level": 0.95, "mean": 0.0011547216359594306, "replicates": 12, "sample_sd": 0.0026161911154972406, "standard_error": 0.0007552293223919196 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 8.673794285278305e-06, 0.00032856853746742316 ], "confidence_level": 0.95, "mean": 0.00014584370965250483, "replicates": 12, "sample_sd": 0.0003024697008993026, "standard_error": 8.731548161795897e-05 }, "log_loss": { "bootstrap_interval": [ 0.00010095064078535312, 0.0028066810876011747 ], "confidence_level": 0.95, "mean": 0.0011547216359594306, "replicates": 12, "sample_sd": 0.002616191115497242, "standard_error": 0.00075522932239192 } } }, "discount": { "bootstrap_interval": [ 0.8996720210670056, 0.9856211723088635 ], "confidence_level": 0.95, "mean": 0.9461240561831215, "replicates": 12, "sample_sd": 0.07863795467940007, "standard_error": 0.022700822151336612 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7651424940530456, "discount_one_count": 7, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16189211833950326, 0.16283668435685614 ], "confidence_level": 0.95, "mean": 0.16236605327424372, "replicates": 12, "sample_sd": 0.0008752554684856174, "standard_error": 0.0002526644901699316 }, "log_loss": { "bootstrap_interval": [ 0.5120557158136242, 0.5140556495623329 ], "confidence_level": 0.95, "mean": 0.5130507765650558, "replicates": 12, "sample_sd": 0.001849287610909243, "standard_error": 0.0005338433499837456 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.03988910974046612, 0.04300391347336836 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.002919907694614284, "standard_error": 0.0008429047467472082 }, "log_loss": { "bootstrap_interval": [ 0.13716995816538607, 0.1472738916208268 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.03983913208761096, 0.04301014153464473 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.0029199076946142856, "standard_error": 0.0008429047467472088 }, "log_loss": { "bootstrap_interval": [ 0.1371224287694655, 0.1473000291225913 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244682, "standard_error": 0.0027242944314927087 } }, "independent": { "brier": { "bootstrap_interval": [ 0.03988673429739867, 0.04301072493831162 ], "confidence_level": 0.95, "mean": 0.041370450534447815, "replicates": 12, "sample_sd": 0.002919907694614284, "standard_error": 0.0008429047467472082 }, "log_loss": { "bootstrap_interval": [ 0.137123733395864, 0.14735903009961906 ], "confidence_level": 0.95, "mean": 0.14209097416447353, "replicates": 12, "sample_sd": 0.009437232740244689, "standard_error": 0.0027242944314927104 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1947639296411587, 0.2007669581034877 ], "confidence_level": 0.95, "mean": 0.19798151872987868, "replicates": 12, "sample_sd": 0.005536209637648682, "standard_error": 0.0015981660622933337 }, "log_loss": { "bootstrap_interval": [ 0.5721962399050738, 0.5864420645764051 ], "confidence_level": 0.95, "mean": 0.5798492083256285, "replicates": 12, "sample_sd": 0.013175692779153196, "standard_error": 0.0038034948864019533 } }, "learned": { "brier": { "bootstrap_interval": [ 0.040043444052446905, 0.04310216345070172 ], "confidence_level": 0.95, "mean": 0.041516294244100325, "replicates": 12, "sample_sd": 0.0028432879231083713, "standard_error": 0.0008207865238951151 }, "log_loss": { "bootstrap_interval": [ 0.13875230449649717, 0.14805731407332526 ], "confidence_level": 0.95, "mean": 0.14324569580043295, "replicates": 12, "sample_sd": 0.008633876235227687, "standard_error": 0.002492385384279309 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-9-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.020375363887724648, -0.017121799111184492 ], "confidence_level": 0.95, "mean": -0.01878395187580244, "replicates": 12, "sample_sd": 0.003004191271862184, "standard_error": 0.0008672353197533781 }, "log_loss": { "bootstrap_interval": [ -0.06042878060782948, -0.05019898154708789 ], "confidence_level": 0.95, "mean": -0.05552465735276924, "replicates": 12, "sample_sd": 0.009444219855918378, "standard_error": 0.0027263114380502423 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.016941459780597475, 0.020088325362864926 ], "confidence_level": 0.95, "mean": 0.01849284336724136, "replicates": 12, "sample_sd": 0.0029197581202223353, "standard_error": 0.0008428615683394805 }, "log_loss": { "bootstrap_interval": [ 0.046059790319421465, 0.05406795760525234 ], "confidence_level": 0.95, "mean": 0.04997351813022571, "replicates": 12, "sample_sd": 0.007480645067494927, "standard_error": 0.0021594762217151215 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03823863148322264, -0.0339304994051294 ], "confidence_level": 0.95, "mean": -0.0362138320997529, "replicates": 12, "sample_sd": 0.003927568523933967, "standard_error": 0.0011337913722769884 }, "log_loss": { "bootstrap_interval": [ -0.530331447692805, -0.47439642386786024 ], "confidence_level": 0.95, "mean": -0.5045629973505986, "replicates": 12, "sample_sd": 0.05226019059807599, "standard_error": 0.015086217554850161 } } }, "discount": { "bootstrap_interval": [ 0.21323258343671458, 0.24270558094126088 ], "confidence_level": 0.95, "mean": 0.22778332988397162, "replicates": 12, "sample_sd": 0.027511249078736574, "standard_error": 0.007941813530675704 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.27575524291577636, "discount_min": 0.1890704150503637, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1746547989375796, 0.17795823252653528 ], "confidence_level": 0.95, "mean": 0.17632859808053702, "replicates": 12, "sample_sd": 0.0030490679565967974, "standard_error": 0.0008801901027593116 }, "log_loss": { "bootstrap_interval": [ 0.5314667511192819, 0.5385820596996175 ], "confidence_level": 0.95, "mean": 0.5350970878959407, "replicates": 12, "sample_sd": 0.006658797657263876, "standard_error": 0.0019222293099502743 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1547894420623562, 0.16022226972006934 ], "confidence_level": 0.95, "mean": 0.1575949377509461, "replicates": 12, "sample_sd": 0.0050174332477409735, "standard_error": 0.0014484082181121147 }, "log_loss": { "bootstrap_interval": [ 0.4731793265124728, 0.4863972953489571 ], "confidence_level": 0.95, "mean": 0.47994417193496464, "replicates": 12, "sample_sd": 0.012461663802208237, "standard_error": 0.0035973724753777707 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.13581076712902418, 0.14227920800030175 ], "confidence_level": 0.95, "mean": 0.13905180283749322, "replicates": 12, "sample_sd": 0.006070248360802047, "standard_error": 0.0017523297625784734 }, "log_loss": { "bootstrap_interval": [ 0.4212511931489842, 0.4378717587593232 ], "confidence_level": 0.95, "mean": 0.4295989124129458, "replicates": 12, "sample_sd": 0.015536415239947358, "standard_error": 0.004484976760512706 } }, "independent": { "brier": { "bootstrap_interval": [ 0.18857762278410564, 0.19871440749476194 ], "confidence_level": 0.95, "mean": 0.19375847830448747, "replicates": 12, "sample_sd": 0.009385160985798958, "standard_error": 0.002709262610769501 }, "log_loss": { "bootstrap_interval": [ 0.9461385174769578, 1.0187629443480697 ], "confidence_level": 0.95, "mean": 0.9841354278937702, "replicates": 12, "sample_sd": 0.06651235008484067, "standard_error": 0.019200461612958696 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19616699118462871, 0.20000850873925755 ], "confidence_level": 0.95, "mean": 0.198204636536768, "replicates": 12, "sample_sd": 0.003599045641874691, "standard_error": 0.0010389549850810512 }, "log_loss": { "bootstrap_interval": [ 0.5749887816862919, 0.5846414728074665 ], "confidence_level": 0.95, "mean": 0.5800455524940334, "replicates": 12, "sample_sd": 0.00908889936602059, "standard_error": 0.0026237392478047033 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15436402848223976, 0.16076065886693852 ], "confidence_level": 0.95, "mean": 0.15754464620473457, "replicates": 12, "sample_sd": 0.005911477886383677, "standard_error": 0.0017064966745060678 }, "log_loss": { "bootstrap_interval": [ 0.4710809791745853, 0.4883386369475663 ], "confidence_level": 0.95, "mean": 0.4795724305431715, "replicates": 12, "sample_sd": 0.01579486503161961, "standard_error": 0.004559584788909695 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-9-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.002923299449622556, -0.0012638704863017743 ], "confidence_level": 0.95, "mean": -0.002165537523956533, "replicates": 12, "sample_sd": 0.001533061257599846, "standard_error": 0.0004425566648797287 }, "log_loss": { "bootstrap_interval": [ -0.008243841720691153, -0.003030854233970545 ], "confidence_level": 0.95, "mean": -0.0058588268951483835, "replicates": 12, "sample_sd": 0.0048829611229459485, "standard_error": 0.0014095894593876604 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.025458801154513388, 0.02852648186218412 ], "confidence_level": 0.95, "mean": 0.026966278160804172, "replicates": 12, "sample_sd": 0.0028395136764193126, "standard_error": 0.0008196969927241571 }, "log_loss": { "bootstrap_interval": [ 0.06567144129385206, 0.0724620042902072 ], "confidence_level": 0.95, "mean": 0.0690604358896698, "replicates": 12, "sample_sd": 0.006360609905530089, "standard_error": 0.0018361499205839986 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06298614951427571, -0.057199653730725206 ], "confidence_level": 0.95, "mean": -0.06011344808961661, "replicates": 12, "sample_sd": 0.00534345843903146, "standard_error": 0.0015425235840891957 }, "log_loss": { "bootstrap_interval": [ -1.0682562984083108, -0.9693839804240643 ], "confidence_level": 0.95, "mean": -1.0212820717637023, "replicates": 12, "sample_sd": 0.09215708912565944, "standard_error": 0.026603460107215905 } } }, "discount": { "bootstrap_interval": [ 0.1265433616865778, 0.14940057013833344 ], "confidence_level": 0.95, "mean": 0.13807437813362458, "replicates": 12, "sample_sd": 0.021241047413806353, "standard_error": 0.0061317622211153515 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.16604299957758523, "discount_min": 0.10628764120573142, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18569669246644432, 0.1904104633099854 ], "confidence_level": 0.95, "mean": 0.1880807356686864, "replicates": 12, "sample_sd": 0.004378900181835005, "standard_error": 0.0012640795993684709 }, "log_loss": { "bootstrap_interval": [ 0.551807616241632, 0.5625450909204524 ], "confidence_level": 0.95, "mean": 0.557239364084915, "replicates": 12, "sample_sd": 0.009925330372747142, "standard_error": 0.0028651960812507656 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.18323060139582006, 0.18849513539474444 ], "confidence_level": 0.95, "mean": 0.18586926308150367, "replicates": 12, "sample_sd": 0.004884173569607011, "standard_error": 0.0014099394625907318 }, "log_loss": { "bootstrap_interval": [ 0.544669254476398, 0.5572488853216719 ], "confidence_level": 0.95, "mean": 0.5510553677365796, "replicates": 12, "sample_sd": 0.0115195861819537, "standard_error": 0.0033254180915520313 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15580414772307408, 0.1622772722670452 ], "confidence_level": 0.95, "mean": 0.1589489199839257, "replicates": 12, "sample_sd": 0.006076820269814485, "standard_error": 0.0017542269092971838 }, "log_loss": { "bootstrap_interval": [ 0.47437430728522606, 0.4903903399058405 ], "confidence_level": 0.95, "mean": 0.48232010130009695, "replicates": 12, "sample_sd": 0.014652651746984206, "standard_error": 0.004229856215231586 } }, "independent": { "brier": { "bootstrap_interval": [ 0.24038815994041643, 0.25175577836339963 ], "confidence_level": 0.95, "mean": 0.2460286462343465, "replicates": 12, "sample_sd": 0.0104563255721727, "standard_error": 0.003018481191914138 }, "log_loss": { "bootstrap_interval": [ 1.514870338873543, 1.6258460385664393 ], "confidence_level": 0.95, "mean": 1.5726626089534692, "replicates": 12, "sample_sd": 0.1044482173907656, "standard_error": 0.030151603213467534 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19597925317961767, 0.2007447998039511 ], "confidence_level": 0.95, "mean": 0.19838127447040257, "replicates": 12, "sample_sd": 0.004438570010571056, "standard_error": 0.0012813047952100998 }, "log_loss": { "bootstrap_interval": [ 0.5740765185138585, 0.5858714680360086 ], "confidence_level": 0.95, "mean": 0.5802357830613853, "replicates": 12, "sample_sd": 0.011006438851788843, "standard_error": 0.0031772852169497223 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1830075087575034, 0.1890953845943371 ], "confidence_level": 0.95, "mean": 0.1859151981447299, "replicates": 12, "sample_sd": 0.005579373504731297, "standard_error": 0.00161062639743304 }, "log_loss": { "bootstrap_interval": [ 0.5444135021085487, 0.5589255465304203 ], "confidence_level": 0.95, "mean": 0.5513805371897667, "replicates": 12, "sample_sd": 0.013832017456604798, "standard_error": 0.003992959501003192 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12195121951219513, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.13368983957219252, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 }, "groups-9-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0012142099324144154, 0.00010918961875158317 ], "confidence_level": 0.95, "mean": -0.0005674168364761072, "replicates": 12, "sample_sd": 0.0012255278096101297, "standard_error": 0.00035377940538889045 }, "log_loss": { "bootstrap_interval": [ -0.0028711642097383407, 0.0014836414630705816 ], "confidence_level": 0.95, "mean": -0.0007627865898868306, "replicates": 12, "sample_sd": 0.004014322613089127, "standard_error": 0.0011588351206405047 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.0270613324726238, 0.029786929251269037 ], "confidence_level": 0.95, "mean": 0.02842136310306729, "replicates": 12, "sample_sd": 0.00251613739357825, "standard_error": 0.0007263463007502431 }, "log_loss": { "bootstrap_interval": [ 0.06926967918176367, 0.07455494584544929 ], "confidence_level": 0.95, "mean": 0.0720106167703841, "replicates": 12, "sample_sd": 0.005003014747162297, "standard_error": 0.00144424595551691 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06848502552791763, -0.061949566927803124 ], "confidence_level": 0.95, "mean": -0.06518354400733126, "replicates": 12, "sample_sd": 0.0061309840325157895, "standard_error": 0.0017698626407851442 }, "log_loss": { "bootstrap_interval": [ -1.1965865071844117, -1.0859574766166475 ], "confidence_level": 0.95, "mean": -1.1442558930636277, "replicates": 12, "sample_sd": 0.10185347716049264, "standard_error": 0.029402566228254916 } } }, "discount": { "bootstrap_interval": [ 0.11320135354855, 0.13579004640920272 ], "confidence_level": 0.95, "mean": 0.12468271906333259, "replicates": 12, "sample_sd": 0.021012922180368997, "standard_error": 0.00606590813864835 }, "discount_below_exchangeable_range_count": 3, "discount_max": 0.15504516973122634, "discount_min": 0.09302476784286728, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18845695826073253, 0.19363734720776518 ], "confidence_level": 0.95, "mean": 0.19108910017750316, "replicates": 12, "sample_sd": 0.004736600505090449, "standard_error": 0.001367338788328844 }, "log_loss": { "bootstrap_interval": [ 0.5576244761615211, 0.5693689039605783 ], "confidence_level": 0.95, "mean": 0.5634366276950702, "replicates": 12, "sample_sd": 0.010848766390647748, "standard_error": 0.003131769098007921 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.18781719046223933, 0.1930441175778118 ], "confidence_level": 0.95, "mean": 0.190387296431224, "replicates": 12, "sample_sd": 0.004873418512023042, "standard_error": 0.0014068347448951044 }, "log_loss": { "bootstrap_interval": [ 0.555782418351992, 0.5680523138670672 ], "confidence_level": 0.95, "mean": 0.561989873062894, "replicates": 12, "sample_sd": 0.011409386326503986, "standard_error": 0.003293606133447756 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15919712131344682, 0.1652069694095165 ], "confidence_level": 0.95, "mean": 0.16210032023795976, "replicates": 12, "sample_sd": 0.005645383574176981, "standard_error": 0.0016296818631148858 }, "log_loss": { "bootstrap_interval": [ 0.4835980396257592, 0.49813314635228334 ], "confidence_level": 0.95, "mean": 0.4906632243347993, "replicates": 12, "sample_sd": 0.013520279358365025, "standard_error": 0.003902968463535494 } }, "independent": { "brier": { "bootstrap_interval": [ 0.24991730028945208, 0.2617873442375639 ], "confidence_level": 0.95, "mean": 0.2557052273483583, "replicates": 12, "sample_sd": 0.01106139282035812, "standard_error": 0.0031931490612229774 }, "log_loss": { "bootstrap_interval": [ 1.643683185604595, 1.7650980061261956 ], "confidence_level": 0.95, "mean": 1.706929734168811, "replicates": 12, "sample_sd": 0.11350478901271417, "standard_error": 0.0327660102454011 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19551923491812145, 0.20113898103667915 ], "confidence_level": 0.95, "mean": 0.19836684545746547, "replicates": 12, "sample_sd": 0.005192545167919238, "standard_error": 0.0014989586752387314 }, "log_loss": { "bootstrap_interval": [ 0.57320715193585, 0.586684248657901 ], "confidence_level": 0.95, "mean": 0.5801052504343464, "replicates": 12, "sample_sd": 0.012584439671942404, "standard_error": 0.0036328148160982765 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18774340513627466, 0.19351768293904298 ], "confidence_level": 0.95, "mean": 0.19052168334102704, "replicates": 12, "sample_sd": 0.005366636103378194, "standard_error": 0.0015492143994640824 }, "log_loss": { "bootstrap_interval": [ 0.5558866283566154, 0.5699999304943711 ], "confidence_level": 0.95, "mean": 0.5626738411051834, "replicates": 12, "sample_sd": 0.013125426264709908, "standard_error": 0.003788984193579425 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1111111111111111, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.12195121951219512, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 200 } }, "config": { "conditions": [ { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 } ], "phase": "exploratory-stress-groups-m200-v1", "replicates": 12, "schema_version": 1, "seed": 84173101, "test_count": 2000, "train_count": 200 }, "config_sha256": "98309ad42737e5cfbce19500bd3de567c348d6bcb03875ca668d175f72eb96f6", "expected_runs": 144, "input_path": "results/stress-v1/stress-groups-m200-v1.json", "input_sha256": "20341ee0a56500c49581d5943c9786cb076a9e8f1c397dfe19957cab40320c8a", "ledger_receipt": { "bytes": 97651528, "count": 316800, "payload_sha256": "3a04dbbd091d3299671d0032b1ab454b799fbc858491da4d2c40e433948c7380", "schema_version": 1, "sha256": "d2be04ab2839814e9a7f5876353fe734d553d2319413aec88bac322fb4c8c145" }, "manifest_sha256": null, "observed_runs": 144, "phase": "exploratory-stress-groups-m200-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" }, "exploratory-stress-groups-m50-v1": { "analysis_seed": 38063205225666835376905953319411741762168349098967955198924985924076375439235, "cells": { "groups-4-3-2-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.120815078728055, -0.11860543171193544 ], "confidence_level": 0.95, "mean": -0.11969144474923898, "replicates": 12, "sample_sd": 0.0020658512393520226, "standard_error": 0.0005963598845728061 }, "log_loss": { "bootstrap_interval": [ -0.37016969242001496, -0.36159813810816804 ], "confidence_level": 0.95, "mean": -0.36588151781882366, "replicates": 12, "sample_sd": 0.007909085342421131, "standard_error": 0.002283156275745282 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.0, 0.0005568437208932457 ], "confidence_level": 0.95, "mean": 0.00020694191421985012, "replicates": 12, "sample_sd": 0.0005239529427621429, "standard_error": 0.0001512521862732099 }, "log_loss": { "bootstrap_interval": [ -3.982769953947418e-06, 0.0028770011316705726 ], "confidence_level": 0.95, "mean": 0.0010745101127428252, "replicates": 12, "sample_sd": 0.002695966529649259, "standard_error": 0.0007782585008096105 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0005568437208932504 ], "confidence_level": 0.95, "mean": 0.00020694191421985188, "replicates": 12, "sample_sd": 0.0005239529427621443, "standard_error": 0.0001512521862732103 }, "log_loss": { "bootstrap_interval": [ -3.982769953947418e-06, 0.0028770011316705812 ], "confidence_level": 0.95, "mean": 0.0010745101127428275, "replicates": 12, "sample_sd": 0.002695966529649262, "standard_error": 0.0007782585008096113 } } }, "discount": { "bootstrap_interval": [ 0.8915049476386292, 1.0 ], "confidence_level": 0.95, "mean": 0.9559111292693457, "replicates": 12, "sample_sd": 0.10142440331436565, "standard_error": 0.029278703277973092 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7289271344334884, "discount_one_count": 9, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1623244246357268, 0.16352062452592458 ], "confidence_level": 0.95, "mean": 0.16292656443544098, "replicates": 12, "sample_sd": 0.0011160382501972542, "standard_error": 0.0003221724920886518 }, "log_loss": { "bootstrap_interval": [ 0.5129294323801301, 0.5154604084755187 ], "confidence_level": 0.95, "mean": 0.5142002802860627, "replicates": 12, "sample_sd": 0.0023365281576891222, "standard_error": 0.0006744975804054776 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.041608261706246714, 0.044493274222951164 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14258968119355356, 0.15213191505011414 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934526, "standard_error": 0.0025554921340211595 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.041623449683903815, 0.04451499374706634 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.002659418627093277, "standard_error": 0.0007677080301201042 }, "log_loss": { "bootstrap_interval": [ 0.14260415774518456, 0.15227461958716812 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934534, "standard_error": 0.002555492134021162 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04163654540771806, 0.04452038000395955 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14258194733960072, 0.15209741583268663 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934526, "standard_error": 0.0025554921340211595 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1976554519028035, 0.20238477059700063 ], "confidence_level": 0.95, "mean": 0.19982833660306096, "replicates": 12, "sample_sd": 0.004475773612815193, "standard_error": 0.001292044550095338 }, "log_loss": { "bootstrap_interval": [ 0.5783078217604936, 0.5902749984810923 ], "confidence_level": 0.95, "mean": 0.5836328287027258, "replicates": 12, "sample_sd": 0.011096676232793312, "standard_error": 0.003203334505056671 } }, "learned": { "brier": { "bootstrap_interval": [ 0.041887267283159356, 0.04465864621225599 ], "confidence_level": 0.95, "mean": 0.043235119686202, "replicates": 12, "sample_sd": 0.002576928933683645, "standard_error": 0.000743895306772394 }, "log_loss": { "bootstrap_interval": [ 0.14338906953692473, 0.15345524830559004 ], "confidence_level": 0.95, "mean": 0.14831876246723902, "replicates": 12, "sample_sd": 0.009246362479203948, "standard_error": 0.0026691949331966278 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0, 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-4-3-2-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07153565786409317, -0.06843393975854643 ], "confidence_level": 0.95, "mean": -0.0699468512433681, "replicates": 12, "sample_sd": 0.002919743694359545, "standard_error": 0.0008428574039515979 }, "log_loss": { "bootstrap_interval": [ -0.20993378779897584, -0.1900133845882284 ], "confidence_level": 0.95, "mean": -0.20143265971661004, "replicates": 12, "sample_sd": 0.018735474216696593, "standard_error": 0.005408465541202536 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.006915282428075021, 0.010440572669365912 ], "confidence_level": 0.95, "mean": 0.008565237843429411, "replicates": 12, "sample_sd": 0.003321054409274883, "standard_error": 0.0009587058285941237 }, "log_loss": { "bootstrap_interval": [ 0.022760674327746453, 0.04397079078513326 ], "confidence_level": 0.95, "mean": 0.03217575200810979, "replicates": 12, "sample_sd": 0.019996232960774383, "standard_error": 0.005772415241340779 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.0072661004428324315, -0.0036887158699013 ], "confidence_level": 0.95, "mean": -0.005584535243643285, "replicates": 12, "sample_sd": 0.0032920243200180725, "standard_error": 0.0009503255636706145 }, "log_loss": { "bootstrap_interval": [ -0.06924536105237716, -0.04572945112345431 ], "confidence_level": 0.95, "mean": -0.05786597117366695, "replicates": 12, "sample_sd": 0.021734670446502704, "standard_error": 0.00627425891651807 } } }, "discount": { "bootstrap_interval": [ 0.48959656518092737, 0.6702746341588508 ], "confidence_level": 0.95, "mean": 0.5762818324274799, "replicates": 12, "sample_sd": 0.16766915656984485, "standard_error": 0.04840191634019872 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.9177145679688754, "discount_min": 0.31226717182795427, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16630359570332598, 0.168478862094527 ], "confidence_level": 0.95, "mean": 0.16736231524257644, "replicates": 12, "sample_sd": 0.0020197606988014887, "standard_error": 0.0005830546915758331 }, "log_loss": { "bootstrap_interval": [ 0.5188523574692846, 0.5235273262287531 ], "confidence_level": 0.95, "mean": 0.5211361004999051, "replicates": 12, "sample_sd": 0.004326955996113241, "standard_error": 0.0012490846045638225 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.12063611705495739, 0.1239037057256727 ], "confidence_level": 0.95, "mean": 0.12224347696904729, "replicates": 12, "sample_sd": 0.0030389489965615473, "standard_error": 0.0008772690106091763 }, "log_loss": { "bootstrap_interval": [ 0.4027085937518984, 0.410331012146325 ], "confidence_level": 0.95, "mean": 0.40642190300928394, "replicates": 12, "sample_sd": 0.007033096650523677, "standard_error": 0.0020302801222082502 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.08664893762651907, 0.09126286127221918 ], "confidence_level": 0.95, "mean": 0.08885022615577895, "replicates": 12, "sample_sd": 0.004234769923239126, "standard_error": 0.0012224727775691203 }, "log_loss": { "bootstrap_interval": [ 0.281157405201058, 0.29480727778038723 ], "confidence_level": 0.95, "mean": 0.2875276887751852, "replicates": 12, "sample_sd": 0.012441425005086612, "standard_error": 0.0035915300378946487 } }, "independent": { "brier": { "bootstrap_interval": [ 0.09997655583886542, 0.10605059216068613 ], "confidence_level": 0.95, "mean": 0.10299999924285165, "replicates": 12, "sample_sd": 0.005693404406048032, "standard_error": 0.0016435442832186163 }, "log_loss": { "bootstrap_interval": [ 0.3653230518149464, 0.390316018895431 ], "confidence_level": 0.95, "mean": 0.3775694119569619, "replicates": 12, "sample_sd": 0.022898247109577465, "standard_error": 0.0066101545663425606 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1986871618043729, 0.2043338640625704 ], "confidence_level": 0.95, "mean": 0.20151280964246487, "replicates": 12, "sample_sd": 0.0052547376590107945, "standard_error": 0.001516912100975373 }, "log_loss": { "bootstrap_interval": [ 0.5809558883466753, 0.5937046713768495 ], "confidence_level": 0.95, "mean": 0.5872263762082574, "replicates": 12, "sample_sd": 0.011789209653528695, "standard_error": 0.0034032516834988633 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09484805054415002, 0.09995065339714278 ], "confidence_level": 0.95, "mean": 0.09741546399920836, "replicates": 12, "sample_sd": 0.004758136047577, "standard_error": 0.0013735555639547215 }, "log_loss": { "bootstrap_interval": [ 0.3095621953122134, 0.3328812979247941 ], "confidence_level": 0.95, "mean": 0.319703440783295, "replicates": 12, "sample_sd": 0.021923932272139603, "standard_error": 0.006328894099507462 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.4, 0.5, 0.6666666666666666, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-4-3-2-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.048121070258292965, -0.044633857174096785 ], "confidence_level": 0.95, "mean": -0.04641656061341705, "replicates": 12, "sample_sd": 0.0032209661922905724, "standard_error": 0.000929812849084823 }, "log_loss": { "bootstrap_interval": [ -0.13887997405855065, -0.12379862364782221 ], "confidence_level": 0.95, "mean": -0.1318552139296008, "replicates": 12, "sample_sd": 0.013988038411153017, "standard_error": 0.0040379988710570105 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.012823433682873722, 0.016834895326178732 ], "confidence_level": 0.95, "mean": 0.014751083309338577, "replicates": 12, "sample_sd": 0.003702968277943789, "standard_error": 0.001068954866035746 }, "log_loss": { "bootstrap_interval": [ 0.03880491836132312, 0.055850013497389245 ], "confidence_level": 0.95, "mean": 0.04696854064245533, "replicates": 12, "sample_sd": 0.01604620430922225, "standard_error": 0.0046321401887006 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.016039917509922858, -0.013397460478192805 ], "confidence_level": 0.95, "mean": -0.014725902869199383, "replicates": 12, "sample_sd": 0.002464372165773909, "standard_error": 0.0007114029666464938 }, "log_loss": { "bootstrap_interval": [ -0.19221376394176934, -0.16448945400346696 ], "confidence_level": 0.95, "mean": -0.1784744505623887, "replicates": 12, "sample_sd": 0.0260317008523272, "standard_error": 0.00751470474727746 } } }, "discount": { "bootstrap_interval": [ 0.3230079455351616, 0.43136488392835975 ], "confidence_level": 0.95, "mean": 0.37561098100349977, "replicates": 12, "sample_sd": 0.10129236217660786, "standard_error": 0.02924058628475881 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.549747015136141, "discount_min": 0.23977753058344575, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1693391954487979, 0.17206614619153243 ], "confidence_level": 0.95, "mean": 0.17064811796939722, "replicates": 12, "sample_sd": 0.002515237298034473, "standard_error": 0.000726086465547995 }, "log_loss": { "bootstrap_interval": [ 0.5234872280736352, 0.5293499077880236 ], "confidence_level": 0.95, "mean": 0.5262933145262885, "replicates": 12, "sample_sd": 0.005427260708541954, "standard_error": 0.001566715215519488 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.15652044846756086, 0.15969004680489035 ], "confidence_level": 0.95, "mean": 0.15806335659471601, "replicates": 12, "sample_sd": 0.002924664520517893, "standard_error": 0.0008442779241051768 }, "log_loss": { "bootstrap_interval": [ 0.49266332611185304, 0.49954080837625586 ], "confidence_level": 0.95, "mean": 0.4960069193548253, "replicates": 12, "sample_sd": 0.006466280186895223, "standard_error": 0.0018666543032797504 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.10720070401135688, 0.11203961966577297 ], "confidence_level": 0.95, "mean": 0.10948047404664159, "replicates": 12, "sample_sd": 0.004495176956760734, "standard_error": 0.0012976458130204063 }, "log_loss": { "bootstrap_interval": [ 0.3406984741089196, 0.3552665521104002 ], "confidence_level": 0.95, "mean": 0.3474695599542323, "replicates": 12, "sample_sd": 0.013660069413849202, "standard_error": 0.003943322376617406 } }, "independent": { "brier": { "bootstrap_interval": [ 0.13542082312575265, 0.14263803387621016 ], "confidence_level": 0.95, "mean": 0.13895746022517955, "replicates": 12, "sample_sd": 0.006714072142827176, "standard_error": 0.0019381856795099189 }, "log_loss": { "bootstrap_interval": [ 0.5535628037617718, 0.5920491159859627 ], "confidence_level": 0.95, "mean": 0.5729125511590764, "replicates": 12, "sample_sd": 0.035311358537930784, "standard_error": 0.010193511178662865 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19826228896247425, 0.20495217430758986 ], "confidence_level": 0.95, "mean": 0.20146580021135846, "replicates": 12, "sample_sd": 0.006140024846016372, "standard_error": 0.0017724724988392717 }, "log_loss": { "bootstrap_interval": [ 0.5798099296281068, 0.5950122432900022 ], "confidence_level": 0.95, "mean": 0.587148978169451, "replicates": 12, "sample_sd": 0.014183840809305103, "standard_error": 0.0040945221546975505 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12152630408740722, 0.12724463005102163 ], "confidence_level": 0.95, "mean": 0.12423155735598017, "replicates": 12, "sample_sd": 0.0053530029413307425, "standard_error": 0.0015452788445750815 }, "log_loss": { "bootstrap_interval": [ 0.3854858116983189, 0.4047397683125372 ], "confidence_level": 0.95, "mean": 0.3944381005966876, "replicates": 12, "sample_sd": 0.017897487997736102, "standard_error": 0.005166559756655518 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.27027027027027023, 0.35714285714285715, 0.5263157894736842, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.35714285714285715, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-4-3-2-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04408221391306065, -0.04038736037901154 ], "confidence_level": 0.95, "mean": -0.04220223090985924, "replicates": 12, "sample_sd": 0.003419632662182426, "standard_error": 0.00098716291902033 }, "log_loss": { "bootstrap_interval": [ -0.126692101878115, -0.11281527601650874 ], "confidence_level": 0.95, "mean": -0.12001196144285188, "replicates": 12, "sample_sd": 0.012865282001456058, "standard_error": 0.0037138870133705515 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.014436327318488982, 0.017963553183362155 ], "confidence_level": 0.95, "mean": 0.01614668267195426, "replicates": 12, "sample_sd": 0.003308792909734605, "standard_error": 0.0009551662385639997 }, "log_loss": { "bootstrap_interval": [ 0.042646823856573306, 0.057104695536550766 ], "confidence_level": 0.95, "mean": 0.04965108787061983, "replicates": 12, "sample_sd": 0.01348421405423876, "standard_error": 0.003892557307012642 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.01847201375592946, -0.015380584090523421 ], "confidence_level": 0.95, "mean": -0.01708390729112217, "replicates": 12, "sample_sd": 0.0028660502899981264, "standard_error": 0.0008273574532207116 }, "log_loss": { "bootstrap_interval": [ -0.22725507925902702, -0.1961073642831465 ], "confidence_level": 0.95, "mean": -0.21234125603857137, "replicates": 12, "sample_sd": 0.028460332088259065, "standard_error": 0.008215790196191258 } } }, "discount": { "bootstrap_interval": [ 0.29061736091401014, 0.38156153038067175 ], "confidence_level": 0.95, "mean": 0.3353801661829962, "replicates": 12, "sample_sd": 0.08338969593809495, "standard_error": 0.024072531698750083 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.48625375938232906, "discount_min": 0.2069163514822976, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17003350322107014, 0.17291216005489035 ], "confidence_level": 0.95, "mean": 0.17140030589279143, "replicates": 12, "sample_sd": 0.002676294608417168, "standard_error": 0.000772579706300198 }, "log_loss": { "bootstrap_interval": [ 0.5244208390946501, 0.5307355587792917 ], "confidence_level": 0.95, "mean": 0.5274528721147889, "replicates": 12, "sample_sd": 0.005753785927954417, "standard_error": 0.0016609749271819818 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.16262492344825275, 0.16581666453669022 ], "confidence_level": 0.95, "mean": 0.1641789350105973, "replicates": 12, "sample_sd": 0.0029408108560773556, "standard_error": 0.0008489389696960175 }, "log_loss": { "bootstrap_interval": [ 0.506952656542316, 0.5139497810299366 ], "confidence_level": 0.95, "mean": 0.5103340935173623, "replicates": 12, "sample_sd": 0.006429968742503669, "standard_error": 0.0018561720921826865 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.11075094168006468, 0.11564150647597293 ], "confidence_level": 0.95, "mean": 0.11305139231097792, "replicates": 12, "sample_sd": 0.004529586923612528, "standard_error": 0.001307579114832751 }, "log_loss": { "bootstrap_interval": [ 0.3512170863125305, 0.36539569170854647 ], "confidence_level": 0.95, "mean": 0.3577898228013172, "replicates": 12, "sample_sd": 0.013340391747629328, "standard_error": 0.0038510393832944276 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1424375203169611, 0.15036433336182006 ], "confidence_level": 0.95, "mean": 0.14628198227405437, "replicates": 12, "sample_sd": 0.007324071714544563, "standard_error": 0.0021142773879782137 }, "log_loss": { "bootstrap_interval": [ 0.5989624000730244, 0.6405925803803524 ], "confidence_level": 0.95, "mean": 0.6197821667105083, "replicates": 12, "sample_sd": 0.0393542711490302, "standard_error": 0.011360599520827056 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19751849306573682, 0.2045573542961006 ], "confidence_level": 0.95, "mean": 0.2009001167189986, "replicates": 12, "sample_sd": 0.0065075164017905154, "standard_error": 0.0018785581731648295 }, "log_loss": { "bootstrap_interval": [ 0.5781524829082099, 0.594409773495704 ], "confidence_level": 0.95, "mean": 0.5860789157919176, "replicates": 12, "sample_sd": 0.015177516615798966, "standard_error": 0.004381371651880775 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12635563727246985, 0.1322707996146705 ], "confidence_level": 0.95, "mean": 0.12919807498293218, "replicates": 12, "sample_sd": 0.005454385199601978, "standard_error": 0.0015745453816270565 }, "log_loss": { "bootstrap_interval": [ 0.3990464868087114, 0.4168250866081407 ], "confidence_level": 0.95, "mean": 0.407440910671937, "replicates": 12, "sample_sd": 0.016608280530351547, "standard_error": 0.004794397617487643 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25, 0.3333333333333333, 0.5, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-5-5-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12083537509805763, -0.11860062577219439 ], "confidence_level": 0.95, "mean": -0.11969144474923898, "replicates": 12, "sample_sd": 0.0020658512393520226, "standard_error": 0.0005963598845728061 }, "log_loss": { "bootstrap_interval": [ -0.3701105302216623, -0.3615579778233662 ], "confidence_level": 0.95, "mean": -0.36588151781882366, "replicates": 12, "sample_sd": 0.007909085342421131, "standard_error": 0.002283156275745282 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -1.734723475976807e-18, 0.0005568437208932469 ], "confidence_level": 0.95, "mean": 0.00020694191421985014, "replicates": 12, "sample_sd": 0.000523952942762145, "standard_error": 0.0001512521862732105 }, "log_loss": { "bootstrap_interval": [ -3.982769953947418e-06, 0.002878992516647548 ], "confidence_level": 0.95, "mean": 0.0010745101127428252, "replicates": 12, "sample_sd": 0.002695966529649259, "standard_error": 0.0007782585008096105 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0005568437208932504 ], "confidence_level": 0.95, "mean": 0.00020694191421985188, "replicates": 12, "sample_sd": 0.0005239529427621443, "standard_error": 0.0001512521862732103 }, "log_loss": { "bootstrap_interval": [ -3.982769953947418e-06, 0.002878992516647555 ], "confidence_level": 0.95, "mean": 0.0010745101127428275, "replicates": 12, "sample_sd": 0.002695966529649262, "standard_error": 0.0007782585008096113 } } }, "discount": { "bootstrap_interval": [ 0.8915049476386292, 1.0 ], "confidence_level": 0.95, "mean": 0.9559111292693457, "replicates": 12, "sample_sd": 0.10142440331436565, "standard_error": 0.029278703277973092 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7289271344334884, "discount_one_count": 9, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1623043457473176, 0.1635190853962768 ], "confidence_level": 0.95, "mean": 0.16292656443544098, "replicates": 12, "sample_sd": 0.0011160382501972542, "standard_error": 0.0003221724920886518 }, "log_loss": { "bootstrap_interval": [ 0.5129296172308996, 0.515443127306648 ], "confidence_level": 0.95, "mean": 0.5142002802860627, "replicates": 12, "sample_sd": 0.0023365281576891222, "standard_error": 0.0006744975804054776 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.04164733865999011, 0.044501245103944494 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14257680171342726, 0.15223478069606808 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934526, "standard_error": 0.0025554921340211595 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.04162392796605604, 0.044470144649670945 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14264098911723508, 0.15207502665200562 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934534, "standard_error": 0.002555492134021162 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04159316801707742, 0.044490490440850225 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14253909764894282, 0.1520494800952815 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934526, "standard_error": 0.0025554921340211595 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1976389023575006, 0.2024846853174559 ], "confidence_level": 0.95, "mean": 0.19982833660306096, "replicates": 12, "sample_sd": 0.004475773612815193, "standard_error": 0.001292044550095338 }, "log_loss": { "bootstrap_interval": [ 0.5783561978178113, 0.5903212781828684 ], "confidence_level": 0.95, "mean": 0.5836328287027258, "replicates": 12, "sample_sd": 0.011096676232793312, "standard_error": 0.003203334505056671 } }, "learned": { "brier": { "bootstrap_interval": [ 0.0418968405109651, 0.044671756016502445 ], "confidence_level": 0.95, "mean": 0.043235119686202, "replicates": 12, "sample_sd": 0.002576928933683645, "standard_error": 0.000743895306772394 }, "log_loss": { "bootstrap_interval": [ 0.14337984847519178, 0.15325912958059934 ], "confidence_level": 0.95, "mean": 0.14831876246723902, "replicates": 12, "sample_sd": 0.009246362479203948, "standard_error": 0.0026691949331966278 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-5-5-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.043593940227108975, -0.03777114293276983 ], "confidence_level": 0.95, "mean": -0.041169076989125065, "replicates": 12, "sample_sd": 0.005537523168553379, "standard_error": 0.0015985452460040414 }, "log_loss": { "bootstrap_interval": [ -0.12614917765677736, -0.0927675296503761 ], "confidence_level": 0.95, "mean": -0.11339077430996408, "replicates": 12, "sample_sd": 0.03323868658003896, "standard_error": 0.009595182322247547 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.0004177416552376758, 0.004445621060349411 ], "confidence_level": 0.95, "mean": 0.00202179883950416, "replicates": 12, "sample_sd": 0.003948221071219449, "standard_error": 0.0011397532491443508 }, "log_loss": { "bootstrap_interval": [ 0.002411694122449831, 0.03237639458846833 ], "confidence_level": 0.95, "mean": 0.013978352866684488, "replicates": 12, "sample_sd": 0.02954729347805395, "standard_error": 0.008529568921689662 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.018899912416107743, -0.015117294673948968 ], "confidence_level": 0.95, "mean": -0.017123562578205777, "replicates": 12, "sample_sd": 0.0034669770497069053, "standard_error": 0.0010008300664612682 }, "log_loss": { "bootstrap_interval": [ -0.22665316506075736, -0.1918336474636075 ], "confidence_level": 0.95, "mean": -0.20946563405235755, "replicates": 12, "sample_sd": 0.032206426310851646, "standard_error": 0.009297194450103023 } } }, "discount": { "bootstrap_interval": [ 0.3179251891305575, 0.4500623792010438 ], "confidence_level": 0.95, "mean": 0.3760991619050799, "replicates": 12, "sample_sd": 0.12282290138522509, "standard_error": 0.03545591758870528 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.6856817037661298, "discount_min": 0.23290778347183144, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17026027671137398, 0.17264212854759378 ], "confidence_level": 0.95, "mean": 0.17144976854951, "replicates": 12, "sample_sd": 0.0022120274484539373, "standard_error": 0.0006385573214098608 }, "log_loss": { "bootstrap_interval": [ 0.5250635871133669, 0.5302630079916589 ], "confidence_level": 0.95, "mean": 0.5276081753755779, "replicates": 12, "sample_sd": 0.004763130889095353, "standard_error": 0.0013749974505023119 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.13629556611880755, 0.1400155199731714 ], "confidence_level": 0.95, "mean": 0.13811664739888188, "replicates": 12, "sample_sd": 0.003520640868714315, "standard_error": 0.001016321476636104 }, "log_loss": { "bootstrap_interval": [ 0.43331104169966644, 0.4421212764161634 ], "confidence_level": 0.95, "mean": 0.4376421914948541, "replicates": 12, "sample_sd": 0.00814996333696394, "standard_error": 0.002352691763240856 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.12562692118826324, 0.13087010155685277 ], "confidence_level": 0.95, "mean": 0.12825889272088076, "replicates": 12, "sample_sd": 0.004934096465573229, "standard_error": 0.0014243509613031426 }, "log_loss": { "bootstrap_interval": [ 0.3932416480364385, 0.40728730801434554 ], "confidence_level": 0.95, "mean": 0.40023904819892936, "replicates": 12, "sample_sd": 0.013238516723057224, "standard_error": 0.0038216305968642255 } }, "independent": { "brier": { "bootstrap_interval": [ 0.14348873339332538, 0.1515358999629234 ], "confidence_level": 0.95, "mean": 0.1474042541385907, "replicates": 12, "sample_sd": 0.007310443797538364, "standard_error": 0.0021103433472022025 }, "log_loss": { "bootstrap_interval": [ 0.60444985602636, 0.6437294644311479 ], "confidence_level": 0.95, "mean": 0.6236830351179714, "replicates": 12, "sample_sd": 0.03669595684357092, "standard_error": 0.010593210280903282 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1986394939846426, 0.2044360977513492 ], "confidence_level": 0.95, "mean": 0.20151280964246487, "replicates": 12, "sample_sd": 0.0052547376590107945, "standard_error": 0.001516912100975373 }, "log_loss": { "bootstrap_interval": [ 0.5809090996969852, 0.5936445265896743 ], "confidence_level": 0.95, "mean": 0.5872263762082574, "replicates": 12, "sample_sd": 0.011789209653528695, "standard_error": 0.0034032516834988633 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12677741956520452, 0.13484576531032216 ], "confidence_level": 0.95, "mean": 0.1302806915603849, "replicates": 12, "sample_sd": 0.007461845943264456, "standard_error": 0.0021540493819976253 }, "log_loss": { "bootstrap_interval": [ 0.3996378180312221, 0.43696262996632196 ], "confidence_level": 0.95, "mean": 0.41421740106561383, "replicates": 12, "sample_sd": 0.03673705704445346, "standard_error": 0.010605074886924921 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333, 0.3333333333333333 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-5-5-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.019894545309568353, -0.015355487657261465 ], "confidence_level": 0.95, "mean": -0.017909683150282536, "replicates": 12, "sample_sd": 0.0042167281757277955, "standard_error": 0.0012172645736779613 }, "log_loss": { "bootstrap_interval": [ -0.05680157080142939, -0.03808165486198675 ], "confidence_level": 0.95, "mean": -0.04913162022674005, "replicates": 12, "sample_sd": 0.01823646328270665, "standard_error": 0.005264413492668706 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.0004324454504584695, 0.0029395908966053244 ], "confidence_level": 0.95, "mean": 0.0014301359117027633, "replicates": 12, "sample_sd": 0.002411040869359338, "standard_error": 0.0006960075474759016 }, "log_loss": { "bootstrap_interval": [ 0.002000280200511195, 0.015903440723268852 ], "confidence_level": 0.95, "mean": 0.007469572277083008, "replicates": 12, "sample_sd": 0.013776292705392119, "standard_error": 0.003976873150946609 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03695543926160304, -0.033208386567775444 ], "confidence_level": 0.95, "mean": -0.03518825584004694, "replicates": 12, "sample_sd": 0.0034778606639818523, "standard_error": 0.0010039718952769665 }, "log_loss": { "bootstrap_interval": [ -0.528269053577933, -0.4842197054821421 ], "confidence_level": 0.95, "mean": -0.5060451226936519, "replicates": 12, "sample_sd": 0.0405321690662251, "standard_error": 0.011700629360612244 } } }, "discount": { "bootstrap_interval": [ 0.2028085211779041, 0.2609807409083062 ], "confidence_level": 0.95, "mean": 0.2300049266998406, "replicates": 12, "sample_sd": 0.05394772733530851, "standard_error": 0.015573367449604452 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3507634364826432, "discount_min": 0.15778450845178693, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1766244692589189, 0.179748486334226 ], "confidence_level": 0.95, "mean": 0.17813531521553816, "replicates": 12, "sample_sd": 0.002859485907314397, "standard_error": 0.0008254624791659542 }, "log_loss": { "bootstrap_interval": [ 0.5355098603057501, 0.5423500733537241 ], "confidence_level": 0.95, "mean": 0.5389362344603045, "replicates": 12, "sample_sd": 0.006302164111255382, "standard_error": 0.0018192780730552469 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.16902018115798556, 0.17252620270959484 ], "confidence_level": 0.95, "mean": 0.17073511534379243, "replicates": 12, "sample_sd": 0.0032550850351488916, "standard_error": 0.0009396621106391676 }, "log_loss": { "bootstrap_interval": [ 0.5158715183460488, 0.5238811908610912 ], "confidence_level": 0.95, "mean": 0.5198084830302043, "replicates": 12, "sample_sd": 0.00737256663977232, "standard_error": 0.002128276667045502 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15617918412372533, 0.16156511124879577 ], "confidence_level": 0.95, "mean": 0.15879549615355287, "replicates": 12, "sample_sd": 0.004940921872634581, "standard_error": 0.0014263212866052427 }, "log_loss": { "bootstrap_interval": [ 0.4755586193833663, 0.4896193073628684 ], "confidence_level": 0.95, "mean": 0.4823350419564814, "replicates": 12, "sample_sd": 0.012963909246480535, "standard_error": 0.0037423582466027078 } }, "independent": { "brier": { "bootstrap_interval": [ 0.19114031600473588, 0.19986013035649428 ], "confidence_level": 0.95, "mean": 0.19541388790530256, "replicates": 12, "sample_sd": 0.008145590923059831, "standard_error": 0.002351429556068583 }, "log_loss": { "bootstrap_interval": [ 0.9653303422558119, 1.0278563180032374 ], "confidence_level": 0.95, "mean": 0.9958497369272163, "replicates": 12, "sample_sd": 0.05794912753695451, "standard_error": 0.01672847219138232 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19827488117979272, 0.20491026352213954 ], "confidence_level": 0.95, "mean": 0.20146580021135846, "replicates": 12, "sample_sd": 0.006140024846016372, "standard_error": 0.0017724724988392717 }, "log_loss": { "bootstrap_interval": [ 0.579688414804685, 0.5947755136020194 ], "confidence_level": 0.95, "mean": 0.587148978169451, "replicates": 12, "sample_sd": 0.014183840809305103, "standard_error": 0.0040945221546975505 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1570253471002606, 0.1642097908500594 ], "confidence_level": 0.95, "mean": 0.16022563206525564, "replicates": 12, "sample_sd": 0.0067054882405285935, "standard_error": 0.0019357077203585268 }, "log_loss": { "bootstrap_interval": [ 0.4793200822287113, 0.5042508198242718 ], "confidence_level": 0.95, "mean": 0.4898046142335644, "replicates": 12, "sample_sd": 0.023404104230602036, "standard_error": 0.006756182938840073 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2173913043478261, 0.2173913043478261 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-5-5-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.016115216695786187, -0.011992804994386347 ], "confidence_level": 0.95, "mean": -0.014251668531042342, "replicates": 12, "sample_sd": 0.003830461340288901, "standard_error": 0.001105758942968126 }, "log_loss": { "bootstrap_interval": [ -0.04612088643174502, -0.02958682449688569 ], "confidence_level": 0.95, "mean": -0.03905890498500437, "replicates": 12, "sample_sd": 0.015884193352467678, "standard_error": 0.004585371653953639 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.00042046221777179406, 0.0027236861962389703 ], "confidence_level": 0.95, "mean": 0.0014256883302030561, "replicates": 12, "sample_sd": 0.0021599053810595448, "standard_error": 0.0006235109765894248 }, "log_loss": { "bootstrap_interval": [ 0.001767521730421744, 0.013702498952592788 ], "confidence_level": 0.95, "mean": 0.0067055066391294715, "replicates": 12, "sample_sd": 0.011387389046376404, "standard_error": 0.0032872560656462065 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.041134529064612776, -0.037326815714296586 ], "confidence_level": 0.95, "mean": -0.039141359521265906, "replicates": 12, "sample_sd": 0.003487261284181609, "standard_error": 0.001006685620578406 }, "log_loss": { "bootstrap_interval": [ -0.6052587240393482, -0.5559390339668957 ], "confidence_level": 0.95, "mean": -0.5808495411094553, "replicates": 12, "sample_sd": 0.04658228819157891, "standard_error": 0.013447148313438406 } } }, "discount": { "bootstrap_interval": [ 0.1854659008686195, 0.23497026248027802 ], "confidence_level": 0.95, "mean": 0.2092446995770703, "replicates": 12, "sample_sd": 0.04556757472801689, "standard_error": 0.01315422576776947 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.30455347312007053, "discount_min": 0.13778915972325706, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17813445533440014, 0.18149737998656662 ], "confidence_level": 0.95, "mean": 0.17978085369970367, "replicates": 12, "sample_sd": 0.003080649924220164, "standard_error": 0.000889307031513756 }, "log_loss": { "bootstrap_interval": [ 0.5383209877784425, 0.5456062996178179 ], "confidence_level": 0.95, "mean": 0.5418530638002299, "replicates": 12, "sample_sd": 0.006815040163511791, "standard_error": 0.0019673326364708222 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.17442676323264866, 0.17799334059197658 ], "confidence_level": 0.95, "mean": 0.17612727877670548, "replicates": 12, "sample_sd": 0.003279537997038826, "standard_error": 0.0009467210727039861 }, "log_loss": { "bootstrap_interval": [ 0.5286941634669977, 0.5367732017117663 ], "confidence_level": 0.95, "mean": 0.5326944901094957, "replicates": 12, "sample_sd": 0.007374012411243809, "standard_error": 0.0021286940253196275 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.16145735972081138, 0.16698473789597187 ], "confidence_level": 0.95, "mean": 0.1641034968384583, "replicates": 12, "sample_sd": 0.00503007883952025, "standard_error": 0.0014520586860210284 }, "log_loss": { "bootstrap_interval": [ 0.4893717357766159, 0.503286883836043 ], "confidence_level": 0.95, "mean": 0.49608865217609605, "replicates": 12, "sample_sd": 0.013126870504979236, "standard_error": 0.0037894011098335606 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2003858773650682, 0.20930205628229648 ], "confidence_level": 0.95, "mean": 0.20467054468992724, "replicates": 12, "sample_sd": 0.008313031784659206, "standard_error": 0.002399765569327454 }, "log_loss": { "bootstrap_interval": [ 1.0503841409145946, 1.1192729114418252 ], "confidence_level": 0.95, "mean": 1.0836436999246808, "replicates": 12, "sample_sd": 0.06428265401729268, "standard_error": 0.018556803800553752 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1975875462344559, 0.20461104552154188 ], "confidence_level": 0.95, "mean": 0.2009001167189986, "replicates": 12, "sample_sd": 0.0065075164017905154, "standard_error": 0.0018785581731648295 }, "log_loss": { "bootstrap_interval": [ 0.577996586208847, 0.5945275610694953 ], "confidence_level": 0.95, "mean": 0.5860789157919176, "replicates": 12, "sample_sd": 0.015177516615798966, "standard_error": 0.004381371651880775 } }, "learned": { "brier": { "bootstrap_interval": [ 0.16235825931576248, 0.16937801311498035 ], "confidence_level": 0.95, "mean": 0.16552918516866133, "replicates": 12, "sample_sd": 0.006457736021321251, "standard_error": 0.0018641878151326836 }, "log_loss": { "bootstrap_interval": [ 0.4928964827815539, 0.5156467027272427 ], "confidence_level": 0.95, "mean": 0.5027941588152255, "replicates": 12, "sample_sd": 0.021399620126997032, "standard_error": 0.006177538220438736 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 0.2 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-9-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12082802743625862, -0.11857961163662817 ], "confidence_level": 0.95, "mean": -0.11969144474923898, "replicates": 12, "sample_sd": 0.0020658512393520226, "standard_error": 0.0005963598845728061 }, "log_loss": { "bootstrap_interval": [ -0.3701821624413786, -0.3616810032277663 ], "confidence_level": 0.95, "mean": -0.36588151781882366, "replicates": 12, "sample_sd": 0.007909085342421131, "standard_error": 0.002283156275745282 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -5.782411586589357e-19, 0.0005568437208932498 ], "confidence_level": 0.95, "mean": 0.0002069419142198507, "replicates": 12, "sample_sd": 0.0005239529427621448, "standard_error": 0.00015125218627321045 }, "log_loss": { "bootstrap_interval": [ -3.982769953947418e-06, 0.002532463310089661 ], "confidence_level": 0.95, "mean": 0.0010745101127428252, "replicates": 12, "sample_sd": 0.002695966529649259, "standard_error": 0.0007782585008096105 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ 0.0, 0.0005568437208932504 ], "confidence_level": 0.95, "mean": 0.00020694191421985188, "replicates": 12, "sample_sd": 0.0005239529427621443, "standard_error": 0.0001512521862732103 }, "log_loss": { "bootstrap_interval": [ -3.982769953947418e-06, 0.002532463310089654 ], "confidence_level": 0.95, "mean": 0.0010745101127428275, "replicates": 12, "sample_sd": 0.002695966529649262, "standard_error": 0.0007782585008096113 } } }, "discount": { "bootstrap_interval": [ 0.8915049476386292, 1.0 ], "confidence_level": 0.95, "mean": 0.9559111292693457, "replicates": 12, "sample_sd": 0.10142440331436565, "standard_error": 0.029278703277973092 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.7289271344334884, "discount_one_count": 9, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16231453025795922, 0.16351567251002253 ], "confidence_level": 0.95, "mean": 0.16292656443544098, "replicates": 12, "sample_sd": 0.0011160382501972542, "standard_error": 0.0003221724920886518 }, "log_loss": { "bootstrap_interval": [ 0.5129288715804408, 0.5154327404576889 ], "confidence_level": 0.95, "mean": 0.5142002802860627, "replicates": 12, "sample_sd": 0.0023365281576891222, "standard_error": 0.0006744975804054776 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.04162613612946931, 0.04452245380411049 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14251186441763442, 0.15201497166540165 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934526, "standard_error": 0.0025554921340211595 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.0416035793084425, 0.04448769821884339 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932764, "standard_error": 0.0007677080301201041 }, "log_loss": { "bootstrap_interval": [ 0.14251859039158546, 0.15190856028166783 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934534, "standard_error": 0.002555492134021162 } }, "independent": { "brier": { "bootstrap_interval": [ 0.04162094156100044, 0.04448974312358106 ], "confidence_level": 0.95, "mean": 0.04302817777198215, "replicates": 12, "sample_sd": 0.0026594186270932777, "standard_error": 0.0007677080301201045 }, "log_loss": { "bootstrap_interval": [ 0.14256809240462062, 0.15221180190458966 ], "confidence_level": 0.95, "mean": 0.1472442523544962, "replicates": 12, "sample_sd": 0.008852484428934526, "standard_error": 0.0025554921340211595 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1976671535143441, 0.2025039431691016 ], "confidence_level": 0.95, "mean": 0.19982833660306096, "replicates": 12, "sample_sd": 0.004475773612815193, "standard_error": 0.001292044550095338 }, "log_loss": { "bootstrap_interval": [ 0.5783642806169111, 0.5903207776266567 ], "confidence_level": 0.95, "mean": 0.5836328287027258, "replicates": 12, "sample_sd": 0.011096676232793312, "standard_error": 0.003203334505056671 } }, "learned": { "brier": { "bootstrap_interval": [ 0.04188820615055593, 0.04469152269948305 ], "confidence_level": 0.95, "mean": 0.043235119686202, "replicates": 12, "sample_sd": 0.002576928933683645, "standard_error": 0.000743895306772394 }, "log_loss": { "bootstrap_interval": [ 0.14337338664129096, 0.15340085776625773 ], "confidence_level": 0.95, "mean": 0.14831876246723902, "replicates": 12, "sample_sd": 0.009246362479203948, "standard_error": 0.0026691949331966278 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-9-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.01936185595398981, -0.01275590187000471 ], "confidence_level": 0.95, "mean": -0.01647426662189565, "replicates": 12, "sample_sd": 0.006263483379503657, "standard_error": 0.0018081119076105918 }, "log_loss": { "bootstrap_interval": [ -0.05617372754417582, -0.021713543671164976 ], "confidence_level": 0.95, "mean": -0.04190237346602079, "replicates": 12, "sample_sd": 0.03306842764767848, "standard_error": 0.009546032802032417 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.017859364442882403, 0.023904414669573593 ], "confidence_level": 0.95, "mean": 0.02059555877162186, "replicates": 12, "sample_sd": 0.005648420272611556, "standard_error": 0.0016305584824442107 }, "log_loss": { "bootstrap_interval": [ 0.04812200584370029, 0.07952025545645436 ], "confidence_level": 0.95, "mean": 0.061425845757561263, "replicates": 12, "sample_sd": 0.030049495816604824, "standard_error": 0.008674542249364665 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03699284964909031, -0.032794957878870136 ], "confidence_level": 0.95, "mean": -0.03483797076165919, "replicates": 12, "sample_sd": 0.0038760050943335016, "standard_error": 0.001118906292296904 }, "log_loss": { "bootstrap_interval": [ -0.5183554195565555, -0.47616980515725366 ], "confidence_level": 0.95, "mean": -0.49709318466796315, "replicates": 12, "sample_sd": 0.039182795496324, "standard_error": 0.01131109876370236 } } }, "discount": { "bootstrap_interval": [ 0.19600896071085816, 0.2820360218246135 ], "confidence_level": 0.95, "mean": 0.23566310359559875, "replicates": 12, "sample_sd": 0.08169700194561025, "standard_error": 0.02358389303264173 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.41368814784292285, "discount_min": 0.13254198850247567, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17614971847435026, 0.18045241917048216 ], "confidence_level": 0.95, "mean": 0.1782644298337111, "replicates": 12, "sample_sd": 0.003974722631132445, "standard_error": 0.0011474035905192075 }, "log_loss": { "bootstrap_interval": [ 0.5349547853442962, 0.5441838613216099 ], "confidence_level": 0.95, "mean": 0.5394472313022727, "replicates": 12, "sample_sd": 0.00861066760357812, "standard_error": 0.002485685629414109 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.15673012613859985, 0.16340295659211124 ], "confidence_level": 0.95, "mean": 0.15991774156491853, "replicates": 12, "sample_sd": 0.006085197265302678, "standard_error": 0.0017566451395972379 }, "log_loss": { "bootstrap_interval": [ 0.47804901493360535, 0.49451590475745755 ], "confidence_level": 0.95, "mean": 0.4859912032205737, "replicates": 12, "sample_sd": 0.015038141276015585, "standard_error": 0.00434113745690961 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.13813268871706552, 0.14441428009114649 ], "confidence_level": 0.95, "mean": 0.1411946044401936, "replicates": 12, "sample_sd": 0.0056663576422278225, "standard_error": 0.0016357365550324634 }, "log_loss": { "bootstrap_interval": [ 0.4280026534111492, 0.44433940239353503 ], "confidence_level": 0.95, "mean": 0.4361190120786906, "replicates": 12, "sample_sd": 0.015032272893896505, "standard_error": 0.0043394434009115315 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1916357270294277, 0.20191349323548002 ], "confidence_level": 0.95, "mean": 0.19662813397347464, "replicates": 12, "sample_sd": 0.009529864701847334, "standard_error": 0.002751034975476136 }, "log_loss": { "bootstrap_interval": [ 0.9593952619388731, 1.0301207729723192 ], "confidence_level": 0.95, "mean": 0.994638042504215, "replicates": 12, "sample_sd": 0.06618269134344673, "standard_error": 0.019105297331416443 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19868483655481994, 0.20434938313966367 ], "confidence_level": 0.95, "mean": 0.20151280964246487, "replicates": 12, "sample_sd": 0.0052547376590107945, "standard_error": 0.001516912100975373 }, "log_loss": { "bootstrap_interval": [ 0.581076337302434, 0.5935726810861844 ], "confidence_level": 0.95, "mean": 0.5872263762082574, "replicates": 12, "sample_sd": 0.011789209653528695, "standard_error": 0.0034032516834988633 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15700687949062017, 0.16757618176371092 ], "confidence_level": 0.95, "mean": 0.16179016321181544, "replicates": 12, "sample_sd": 0.009868483520906322, "standard_error": 0.0028487858086443255 }, "log_loss": { "bootstrap_interval": [ 0.47964112845123597, 0.5224832360998363 ], "confidence_level": 0.95, "mean": 0.49754485783625185, "replicates": 12, "sample_sd": 0.04014692467930528, "standard_error": 0.011589418885366267 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-9-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0018345447977226453, 0.005303468748623539 ], "confidence_level": 0.95, "mean": 0.0013687605597908444, "replicates": 12, "sample_sd": 0.0065105413816238826, "standard_error": 0.0018794314096253735 }, "log_loss": { "bootstrap_interval": [ -0.0044689663788645955, 0.027281961377129182 ], "confidence_level": 0.95, "mean": 0.009697634436695884, "replicates": 12, "sample_sd": 0.029598927059579803, "standard_error": 0.00854447425278625 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.026683476915016192, 0.03485455442745682 ], "confidence_level": 0.95, "mean": 0.030443494469268546, "replicates": 12, "sample_sd": 0.007512319838139867, "standard_error": 0.0021686199403943094 }, "log_loss": { "bootstrap_interval": [ 0.06851248515149551, 0.10238825119415632 ], "confidence_level": 0.95, "mean": 0.08336980239047581, "replicates": 12, "sample_sd": 0.031107579571705773, "standard_error": 0.008979984719781016 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06261269941137972, -0.05595837597827831 ], "confidence_level": 0.95, "mean": -0.059271754327428414, "replicates": 12, "sample_sd": 0.006081176036541602, "standard_error": 0.0017554843108433978 }, "log_loss": { "bootstrap_interval": [ -1.0635093855416389, -0.9828759742697496 ], "confidence_level": 0.95, "mean": -1.02303383931788, "replicates": 12, "sample_sd": 0.07435684245245185, "standard_error": 0.021464971503006833 } } }, "discount": { "bootstrap_interval": [ 0.11385782479102796, 0.18081189140399 ], "confidence_level": 0.95, "mean": 0.14537637138847015, "replicates": 12, "sample_sd": 0.06217693005190083, "standard_error": 0.017948933651424737 }, "discount_below_exchangeable_range_count": 1, "discount_max": 0.2671396580213965, "discount_min": 0.06291324480301408, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18753489235688042, 0.19337578572013558 ], "confidence_level": 0.95, "mean": 0.19037213972999562, "replicates": 12, "sample_sd": 0.005387033775110927, "standard_error": 0.0015551027000969499 }, "log_loss": { "bootstrap_interval": [ 0.5560016638243603, 0.5692571032345265 ], "confidence_level": 0.95, "mean": 0.5624667952067893, "replicates": 12, "sample_sd": 0.012065331381000816, "standard_error": 0.0034829611603414304 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.185248622786073, 0.19170809184614335 ], "confidence_level": 0.95, "mean": 0.18833929795270843, "replicates": 12, "sample_sd": 0.005912288616173781, "standard_error": 0.0017067307120373465 }, "log_loss": { "bootstrap_interval": [ 0.5498015628986981, 0.5643662453852144 ], "confidence_level": 0.95, "mean": 0.5568449277941674, "replicates": 12, "sample_sd": 0.013649060750562799, "standard_error": 0.003940144449261494 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.1586181125870811, 0.16427135072967478 ], "confidence_level": 0.95, "mean": 0.16129740582051794, "replicates": 12, "sample_sd": 0.0052684273249308956, "standard_error": 0.0015208639671274162 }, "log_loss": { "bootstrap_interval": [ 0.4816690959238803, 0.49618194132488563 ], "confidence_level": 0.95, "mean": 0.4887946272530094, "replicates": 12, "sample_sd": 0.013609192561879686, "standard_error": 0.003928635494527345 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2445626266788053, 0.2583319145662363 ], "confidence_level": 0.95, "mean": 0.2510126546172149, "replicates": 12, "sample_sd": 0.012720671263115824, "standard_error": 0.003672141489016329 }, "log_loss": { "bootstrap_interval": [ 1.5427129248750775, 1.651075224663814 ], "confidence_level": 0.95, "mean": 1.595198268961365, "replicates": 12, "sample_sd": 0.1005364666188906, "standard_error": 0.029022378032895157 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19822852356619838, 0.20479608906290014 ], "confidence_level": 0.95, "mean": 0.20146580021135846, "replicates": 12, "sample_sd": 0.006140024846016372, "standard_error": 0.0017724724988392717 }, "log_loss": { "bootstrap_interval": [ 0.579819266692359, 0.5949896734192673 ], "confidence_level": 0.95, "mean": 0.587148978169451, "replicates": 12, "sample_sd": 0.014183840809305103, "standard_error": 0.0040945221546975505 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18656716393070766, 0.19806769884034522 ], "confidence_level": 0.95, "mean": 0.19174090028978646, "replicates": 12, "sample_sd": 0.010809760232839485, "standard_error": 0.0031205089901525946 }, "log_loss": { "bootstrap_interval": [ 0.5541885621680623, 0.5945405768263521 ], "confidence_level": 0.95, "mean": 0.5721644296434852, "replicates": 12, "sample_sd": 0.037641439254738965, "standard_error": 0.010866147543204245 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12195121951219513, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.13368983957219252, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 }, "groups-9-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ 2.6581010447629773e-05, 0.007463220965126916 ], "confidence_level": 0.95, "mean": 0.0034030812096510674, "replicates": 12, "sample_sd": 0.006867615903501646, "standard_error": 0.0019825099452888155 }, "log_loss": { "bootstrap_interval": [ 0.0012056569790013843, 0.034796675226122405 ], "confidence_level": 0.95, "mean": 0.01617333932028874, "replicates": 12, "sample_sd": 0.031369659471880126, "standard_error": 0.009055640670238443 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.028551708403975806, 0.037595282421344876 ], "confidence_level": 0.95, "mean": 0.03270754771150545, "replicates": 12, "sample_sd": 0.008380611728884355, "standard_error": 0.002419274218822559 }, "log_loss": { "bootstrap_interval": [ 0.07262267667177202, 0.10889674013241112 ], "confidence_level": 0.95, "mean": 0.08879042435402855, "replicates": 12, "sample_sd": 0.03387335009566193, "standard_error": 0.009778393898042427 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06734564314524451, -0.05961592505102794 ], "confidence_level": 0.95, "mean": -0.06349919796854402, "replicates": 12, "sample_sd": 0.0073798960295410766, "standard_error": 0.0021303924796234955 }, "log_loss": { "bootstrap_interval": [ -1.1953751655203053, -1.1013155828135437 ], "confidence_level": 0.95, "mean": -1.147483462643351, "replicates": 12, "sample_sd": 0.08677590392130337, "standard_error": 0.025050045744068803 } } }, "discount": { "bootstrap_interval": [ 0.10204337265679236, 0.16914861826141875 ], "confidence_level": 0.95, "mean": 0.13381376206262266, "replicates": 12, "sample_sd": 0.06159409251420266, "standard_error": 0.01778068294678281 }, "discount_below_exchangeable_range_count": 5, "discount_max": 0.261461495237706, "discount_min": 0.05177111260492302, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.19047842334727774, 0.19660591456083676 ], "confidence_level": 0.95, "mean": 0.19340796334031268, "replicates": 12, "sample_sd": 0.005733036214724282, "standard_error": 0.0016549850009224688 }, "log_loss": { "bootstrap_interval": [ 0.5620942146226496, 0.5760064939212343 ], "confidence_level": 0.95, "mean": 0.5687057372098359, "replicates": 12, "sample_sd": 0.012915446996703243, "standard_error": 0.0037283684001254806 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1897194348211659, 0.1960098607146507 ], "confidence_level": 0.95, "mean": 0.19279564921831085, "replicates": 12, "sample_sd": 0.005876691578880754, "standard_error": 0.0016964547325056052 }, "log_loss": { "bootstrap_interval": [ 0.5604296910004429, 0.5750081522674677 ], "confidence_level": 0.95, "mean": 0.5675170183800602, "replicates": 12, "sample_sd": 0.013402565484374058, "standard_error": 0.003868987395117475 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.16153539235135592, 0.16688950595180352 ], "confidence_level": 0.95, "mean": 0.1641034968384583, "replicates": 12, "sample_sd": 0.0050300788395202505, "standard_error": 0.0014520586860210286 }, "log_loss": { "bootstrap_interval": [ 0.4890696427147081, 0.5034275735089248 ], "confidence_level": 0.95, "mean": 0.49608865217609605, "replicates": 12, "sample_sd": 0.013126870504979236, "standard_error": 0.0037894011098335606 } }, "independent": { "brier": { "bootstrap_interval": [ 0.25320072963016105, 0.26865044779439684 ], "confidence_level": 0.95, "mean": 0.26031024251850776, "replicates": 12, "sample_sd": 0.014086633582716088, "standard_error": 0.004066460845478378 }, "log_loss": { "bootstrap_interval": [ 1.6739873001303058, 1.79451435992999 ], "confidence_level": 0.95, "mean": 1.7323625391734758, "replicates": 12, "sample_sd": 0.11203408420139332, "standard_error": 0.032341454336043816 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19759337677745495, 0.20458227195157017 ], "confidence_level": 0.95, "mean": 0.2009001167189986, "replicates": 12, "sample_sd": 0.0065075164017905154, "standard_error": 0.0018785581731648295 }, "log_loss": { "bootstrap_interval": [ 0.5780195152081338, 0.5945204906970758 ], "confidence_level": 0.95, "mean": 0.5860789157919176, "replicates": 12, "sample_sd": 0.015177516615798966, "standard_error": 0.004381371651880775 } }, "learned": { "brier": { "bootstrap_interval": [ 0.19140295494300283, 0.2034056744473626 ], "confidence_level": 0.95, "mean": 0.19681104454996373, "replicates": 12, "sample_sd": 0.011177270653117013, "standard_error": 0.003226600110191206 }, "log_loss": { "bootstrap_interval": [ 0.5660632240840922, 0.6083261932794733 ], "confidence_level": 0.95, "mean": 0.5848790765301246, "replicates": 12, "sample_sd": 0.038951415361356496, "standard_error": 0.01124430507209805 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1111111111111111, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.12195121951219512, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 50 } }, "config": { "conditions": [ { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 } ], "phase": "exploratory-stress-groups-m50-v1", "replicates": 12, "schema_version": 1, "seed": 84173100, "test_count": 2000, "train_count": 50 }, "config_sha256": "ff24b68bfafd188e342fc96735bf215ceb9ffb4d29294b82b4b68db17d62c663", "expected_runs": 144, "input_path": "results/stress-v1/stress-groups-m50-v1.json", "input_sha256": "ec158382da9eee32665a99c0337b0f3f5ae9ebc5c1c837e0c7e4adffd9cb795a", "ledger_receipt": { "bytes": 95132081, "count": 295200, "payload_sha256": "80ca2a9e664a0dec0d118ae71d74a93507287248ff182f167af32d5aac900c53", "schema_version": 1, "sha256": "3b78b17e1b8c047e35dae5e3f1b22b0f9b881dd79d32b1756fc5471af2c61945" }, "manifest_sha256": null, "observed_runs": 144, "phase": "exploratory-stress-groups-m50-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" }, "exploratory-stress-groups-m5000-v1": { "analysis_seed": 46468327186324359997129710169848383232840192402147306020094255862160831257704, "cells": { "groups-4-3-2-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12273482279129024, -0.11930867883393392 ], "confidence_level": 0.95, "mean": -0.12100588076871815, "replicates": 12, "sample_sd": 0.003190811622065872, "standard_error": 0.0009211079744665589 }, "log_loss": { "bootstrap_interval": [ -0.37834847730295496, -0.367004007076387 ], "confidence_level": 0.95, "mean": -0.3727523955794017, "replicates": 12, "sample_sd": 0.010359022654777365, "standard_error": 0.002990392259138572 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -6.015124349473845e-06, 6.503528937013131e-06 ], "confidence_level": 0.95, "mean": 6.651527652661028e-07, "replicates": 12, "sample_sd": 1.1017589977404444e-05, "standard_error": 3.1805042696376896e-06 }, "log_loss": { "bootstrap_interval": [ 1.094998992427996e-05, 0.0001669612399086938 ], "confidence_level": 0.95, "mean": 7.134307261333582e-05, "replicates": 12, "sample_sd": 0.00015557896718026247, "standard_error": 4.491177929088425e-05 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -6.015124349475001e-06, 6.503528937011425e-06 ], "confidence_level": 0.95, "mean": 6.651527652649464e-07, "replicates": 12, "sample_sd": 1.101758997740452e-05, "standard_error": 3.1805042696377116e-06 }, "log_loss": { "bootstrap_interval": [ 1.094998992427996e-05, 0.0001669612399086938 ], "confidence_level": 0.95, "mean": 7.134307261333582e-05, "replicates": 12, "sample_sd": 0.00015557896718026247, "standard_error": 4.491177929088425e-05 } } }, "discount": { "bootstrap_interval": [ 0.9816530776703847, 0.9982977761558007 ], "confidence_level": 0.95, "mean": 0.9915007816867969, "replicates": 12, "sample_sd": 0.016032183180547065, "standard_error": 0.004628092637493119 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.9442049583543137, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1612957766443149, 0.16279924001090226 ], "confidence_level": 0.95, "mean": 0.1620594223786907, "replicates": 12, "sample_sd": 0.00140162669241573, "standard_error": 0.00040461477408479325 }, "log_loss": { "bootstrap_interval": [ 0.5107884301934382, 0.5140057868655875 ], "confidence_level": 0.95, "mean": 0.5124084443394333, "replicates": 12, "sample_sd": 0.0029536418712487857, "standard_error": 0.0008526429647276182 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.03897445059431995, 0.04321961001337199 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.13328776902780687, 0.14622648570037985 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.038981159282794275, 0.04322796268664085 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535432, "standard_error": 0.0011389171978370331 }, "log_loss": { "bootstrap_interval": [ 0.13316783649643157, 0.14619221329457832 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "independent": { "brier": { "bootstrap_interval": [ 0.038969376681721686, 0.043261961071728694 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.13322011946828413, 0.14618547751085814 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1970499262576406, 0.19993957553453603 ], "confidence_level": 0.95, "mean": 0.19855599037445573, "replicates": 12, "sample_sd": 0.002670294696841916, "standard_error": 0.0007708476810186552 }, "log_loss": { "bootstrap_interval": [ 0.5775876694643284, 0.5845612123430671 ], "confidence_level": 0.95, "mean": 0.5812378702037866, "replicates": 12, "sample_sd": 0.006406362602030368, "standard_error": 0.0018493575864042921 } }, "learned": { "brier": { "bootstrap_interval": [ 0.03896791369812726, 0.04318802622279701 ], "confidence_level": 0.95, "mean": 0.04105354160997255, "replicates": 12, "sample_sd": 0.003941385178711937, "standard_error": 0.0011377798969546692 }, "log_loss": { "bootstrap_interval": [ 0.13336988412878867, 0.14628993985943525 ], "confidence_level": 0.95, "mean": 0.13965604876003165, "replicates": 12, "sample_sd": 0.012022996752140452, "standard_error": 0.003470740205657143 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0, 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-4-3-2-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.07410372985344983, -0.06991121619935876 ], "confidence_level": 0.95, "mean": -0.07205751200444091, "replicates": 12, "sample_sd": 0.003865224229268699, "standard_error": 0.0011157941246232738 }, "log_loss": { "bootstrap_interval": [ -0.22017524794305487, -0.20854248564689085 ], "confidence_level": 0.95, "mean": -0.21452378642900088, "replicates": 12, "sample_sd": 0.010833881915159833, "standard_error": 0.003127472320043074 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.006406728735818224, 0.0084332519919116 ], "confidence_level": 0.95, "mean": 0.007479733501446814, "replicates": 12, "sample_sd": 0.0018826763346844581, "standard_error": 0.0005434818443135049 }, "log_loss": { "bootstrap_interval": [ 0.019702307573623047, 0.024387125736089636 ], "confidence_level": 0.95, "mean": 0.022232113848407772, "replicates": 12, "sample_sd": 0.004272223392667859, "standard_error": 0.0012332846628975024 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.00770611853871058, -0.005784131460643636 ], "confidence_level": 0.95, "mean": -0.0067116568119326295, "replicates": 12, "sample_sd": 0.0017741553303856396, "standard_error": 0.0005121545287911793 }, "log_loss": { "bootstrap_interval": [ -0.07773556014861563, -0.06494264660413895 ], "confidence_level": 0.95, "mean": -0.07120181098086284, "replicates": 12, "sample_sd": 0.011773912459674116, "standard_error": 0.00339883576400397 } } }, "discount": { "bootstrap_interval": [ 0.49563796382331166, 0.507063257023646 ], "confidence_level": 0.95, "mean": 0.500969749826933, "replicates": 12, "sample_sd": 0.01055877743808442, "standard_error": 0.0030480564980956935 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.5242218335375197, "discount_min": 0.4844189459042447, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1649153305366392, 0.1670905124901544 ], "confidence_level": 0.95, "mean": 0.16597587995158075, "replicates": 12, "sample_sd": 0.002037765425229911, "standard_error": 0.0005882522084009007 }, "log_loss": { "bootstrap_interval": [ 0.5159050115494181, 0.5205019747503489 ], "confidence_level": 0.95, "mean": 0.5181857501580215, "replicates": 12, "sample_sd": 0.0042977450506920326, "standard_error": 0.0012406521309627134 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.11875032240073595, 0.12227701922589337 ], "confidence_level": 0.95, "mean": 0.12050863016663922, "replicates": 12, "sample_sd": 0.0032900549420262553, "standard_error": 0.0009497570532137586 }, "log_loss": { "bootstrap_interval": [ 0.39835685562664136, 0.40634450424427787 ], "confidence_level": 0.95, "mean": 0.4023400091827258, "replicates": 12, "sample_sd": 0.007347023709525991, "standard_error": 0.002120903058218697 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.08387537925117612, 0.08923193131229974 ], "confidence_level": 0.95, "mean": 0.08643863444569305, "replicates": 12, "sample_sd": 0.00497984951093027, "standard_error": 0.0014375587278297088 }, "log_loss": { "bootstrap_interval": [ 0.27413241511511605, 0.2889212569928931 ], "confidence_level": 0.95, "mean": 0.2814298498806128, "replicates": 12, "sample_sd": 0.013591551821693595, "standard_error": 0.003923543051479773 } }, "independent": { "brier": { "bootstrap_interval": [ 0.09694859677259234, 0.10465290974381146 ], "confidence_level": 0.95, "mean": 0.10063002475907248, "replicates": 12, "sample_sd": 0.007276508961363132, "standard_error": 0.002100547203801864 }, "log_loss": { "bootstrap_interval": [ 0.3609396708931895, 0.3889305861396357 ], "confidence_level": 0.95, "mean": 0.3748637747098834, "replicates": 12, "sample_sd": 0.026026101202300474, "standard_error": 0.007513088267552311 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19715086930953632, 0.20205606944479065 ], "confidence_level": 0.95, "mean": 0.19956596318459022, "replicates": 12, "sample_sd": 0.004569486908080526, "standard_error": 0.0013190972482193812 }, "log_loss": { "bootstrap_interval": [ 0.5776884348902823, 0.5890518016324473 ], "confidence_level": 0.95, "mean": 0.5832302241109455, "replicates": 12, "sample_sd": 0.010553555878235293, "standard_error": 0.003046549163603452 } }, "learned": { "brier": { "bootstrap_interval": [ 0.09095140266062543, 0.09704837452957955 ], "confidence_level": 0.95, "mean": 0.09391836794713986, "replicates": 12, "sample_sd": 0.0056400574795067515, "standard_error": 0.0016281443520190927 }, "log_loss": { "bootstrap_interval": [ 0.29600506121506087, 0.31192757283246086 ], "confidence_level": 0.95, "mean": 0.3036619637290206, "replicates": 12, "sample_sd": 0.014604491829431515, "standard_error": 0.004215953644549988 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.4, 0.5, 0.6666666666666666, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.5, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-4-3-2-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.05036420441597589, -0.04620241875873237 ], "confidence_level": 0.95, "mean": -0.04826094338582736, "replicates": 12, "sample_sd": 0.0038366865301634264, "standard_error": 0.0011075560004930327 }, "log_loss": { "bootstrap_interval": [ -0.14778116605821504, -0.1360051203681383 ], "confidence_level": 0.95, "mean": -0.14193652297448323, "replicates": 12, "sample_sd": 0.010867415063588022, "standard_error": 0.0031371525061789697 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.012001429485610705, 0.01514638735143601 ], "confidence_level": 0.95, "mean": 0.01362829118732869, "replicates": 12, "sample_sd": 0.0028787572130306283, "standard_error": 0.0008310256259374051 }, "log_loss": { "bootstrap_interval": [ 0.03494233035108806, 0.042621731436837854 ], "confidence_level": 0.95, "mean": 0.03892398515500626, "replicates": 12, "sample_sd": 0.0070543542405782905, "standard_error": 0.002036416659878427 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.017260625604505562, -0.01437819656187762 ], "confidence_level": 0.95, "mean": -0.015863598196487638, "replicates": 12, "sample_sd": 0.0026585328433060703, "standard_error": 0.0007674523263661104 }, "log_loss": { "bootstrap_interval": [ -0.20195910812756424, -0.17572628681845134 ], "confidence_level": 0.95, "mean": -0.1890137777343602, "replicates": 12, "sample_sd": 0.02411149234597154, "standard_error": 0.0069603882982551355 } } }, "discount": { "bootstrap_interval": [ 0.35458100556706007, 0.3626368783765697 ], "confidence_level": 0.95, "mean": 0.3586259922447576, "replicates": 12, "sample_sd": 0.007509425342310418, "standard_error": 0.002167784371421159 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.37108151443109316, "discount_min": 0.3483163214221986, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.1681497643689811, 0.17069510982078825 ], "confidence_level": 0.95, "mean": 0.16939382821674667, "replicates": 12, "sample_sd": 0.0023657977831741594, "standard_error": 0.0006829469934819105 }, "log_loss": { "bootstrap_interval": [ 0.5209126045391063, 0.5263582843098827 ], "confidence_level": 0.95, "mean": 0.5235584473491358, "replicates": 12, "sample_sd": 0.004993519980883493, "standard_error": 0.0014415050525834298 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.15525577944969762, 0.15831508681600584 ], "confidence_level": 0.95, "mean": 0.15676247496702414, "replicates": 12, "sample_sd": 0.002858267427675491, "standard_error": 0.000825110734392192 }, "log_loss": { "bootstrap_interval": [ 0.4899050885165065, 0.49649218553735647 ], "confidence_level": 0.95, "mean": 0.4930684805080811, "replicates": 12, "sample_sd": 0.0061521342583268, "standard_error": 0.0017759681850678484 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.10529607999051512, 0.10998043628964568 ], "confidence_level": 0.95, "mean": 0.1075045936435906, "replicates": 12, "sample_sd": 0.004389113171768106, "standard_error": 0.001267027835612024 }, "log_loss": { "bootstrap_interval": [ 0.3363209188390139, 0.34951807134009966 ], "confidence_level": 0.95, "mean": 0.3426979392196463, "replicates": 12, "sample_sd": 0.012304484869511774, "standard_error": 0.0035519988258261503 } }, "independent": { "brier": { "bootstrap_interval": [ 0.132354184419503, 0.1414327323666335 ], "confidence_level": 0.95, "mean": 0.13699648302740694, "replicates": 12, "sample_sd": 0.008474097518432568, "standard_error": 0.0024462612417030915 }, "log_loss": { "bootstrap_interval": [ 0.5500449507221478, 0.5917041309666305 ], "confidence_level": 0.95, "mean": 0.5706357021090128, "replicates": 12, "sample_sd": 0.038855036977147525, "standard_error": 0.011216483029064493 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19673502443947752, 0.20324690898200157 ], "confidence_level": 0.95, "mean": 0.1999359132791692, "replicates": 12, "sample_sd": 0.005988078724364737, "standard_error": 0.0017286094317203258 }, "log_loss": { "bootstrap_interval": [ 0.5768054445107903, 0.5918663922331405 ], "confidence_level": 0.95, "mean": 0.5842178182554794, "replicates": 12, "sample_sd": 0.013890520585885505, "standard_error": 0.004009847899722517 } }, "learned": { "brier": { "bootstrap_interval": [ 0.11786413132984373, 0.12435845679591379 ], "confidence_level": 0.95, "mean": 0.1211328848309193, "replicates": 12, "sample_sd": 0.005967640162730889, "standard_error": 0.0017227093271897506 }, "log_loss": { "bootstrap_interval": [ 0.37342670127070504, 0.39013034845638284 ], "confidence_level": 0.95, "mean": 0.3816219243746526, "replicates": 12, "sample_sd": 0.015456432959951019, "standard_error": 0.004461887865069562 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.27027027027027023, 0.35714285714285715, 0.5263157894736842, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.35714285714285715, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-4-3-2-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.045572791531628215, -0.04153917425421944 ], "confidence_level": 0.95, "mean": -0.04353211989278365, "replicates": 12, "sample_sd": 0.0036572905396448354, "standard_error": 0.0010557688387843088 }, "log_loss": { "bootstrap_interval": [ -0.1336335267917398, -0.12228162427436612 ], "confidence_level": 0.95, "mean": -0.127969662116805, "replicates": 12, "sample_sd": 0.010532519048863135, "standard_error": 0.003040476354052996 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.012707087471875314, 0.01653218916157453 ], "confidence_level": 0.95, "mean": 0.014652176499044685, "replicates": 12, "sample_sd": 0.0034795027193096095, "standard_error": 0.001004445915819719 }, "log_loss": { "bootstrap_interval": [ 0.03665068589957329, 0.04594344742581872 ], "confidence_level": 0.95, "mean": 0.041348914441447136, "replicates": 12, "sample_sd": 0.008646081368366657, "standard_error": 0.0024959087027309488 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.01980840799214755, -0.016775470600279865 ], "confidence_level": 0.95, "mean": -0.0183411550914613, "replicates": 12, "sample_sd": 0.0027921975325431105, "standard_error": 0.0008060379985221869 }, "log_loss": { "bootstrap_interval": [ -0.2372259484847505, -0.2084255871337408 ], "confidence_level": 0.95, "mean": -0.2229118239328241, "replicates": 12, "sample_sd": 0.02685345216298037, "standard_error": 0.007751923917483728 } } }, "discount": { "bootstrap_interval": [ 0.3314678462162843, 0.33881941961561546 ], "confidence_level": 0.95, "mean": 0.33519179091979917, "replicates": 12, "sample_sd": 0.006776299074480531, "standard_error": 0.0019561490473803734 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.34585409978908577, "discount_min": 0.32453805942192726, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 4, 3, 2, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16913216554828503, 0.17167781163852575 ], "confidence_level": 0.95, "mean": 0.1703826118497768, "replicates": 12, "sample_sd": 0.0023905783798215514, "standard_error": 0.0006901005355544361 }, "log_loss": { "bootstrap_interval": [ 0.5224844248728537, 0.5280137458852178 ], "confidence_level": 0.95, "mean": 0.5251764316370949, "replicates": 12, "sample_sd": 0.00506443994425402, "standard_error": 0.001461977882554876 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.16171625189835132, 0.1646573361807904 ], "confidence_level": 0.95, "mean": 0.16316443161516073, "replicates": 12, "sample_sd": 0.002696153913136173, "standard_error": 0.0007783125937629163 }, "log_loss": { "bootstrap_interval": [ 0.5049204846982781, 0.5111844903472335 ], "confidence_level": 0.95, "mean": 0.5079954394096052, "replicates": 12, "sample_sd": 0.005794634567204142, "standard_error": 0.0016727669136154112 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.11003705251381304, 0.11443774618389285 ], "confidence_level": 0.95, "mean": 0.11219831545794846, "replicates": 12, "sample_sd": 0.004107161906982825, "standard_error": 0.0011856355163009553 }, "log_loss": { "bootstrap_interval": [ 0.3495136570066057, 0.3623405593388844 ], "confidence_level": 0.95, "mean": 0.3558578550788428, "replicates": 12, "sample_sd": 0.011805720238561223, "standard_error": 0.003408017878855368 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1406226668905071, 0.14955694023107324 ], "confidence_level": 0.95, "mean": 0.14519164704845444, "replicates": 12, "sample_sd": 0.008408963464728742, "standard_error": 0.0024274586599834337 }, "log_loss": { "bootstrap_interval": [ 0.5978831318640552, 0.642049910684806 ], "confidence_level": 0.95, "mean": 0.620118593453114, "replicates": 12, "sample_sd": 0.04111937088208024, "standard_error": 0.011870139923838542 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19663993340725638, 0.2034631152866514 ], "confidence_level": 0.95, "mean": 0.2000848566280008, "replicates": 12, "sample_sd": 0.006311011877348332, "standard_error": 0.0018218322031229926 }, "log_loss": { "bootstrap_interval": [ 0.5765559754683467, 0.5924339293888959 ], "confidence_level": 0.95, "mean": 0.5844790344204306, "replicates": 12, "sample_sd": 0.01460589757934671, "standard_error": 0.004216359449595964 } }, "learned": { "brier": { "bootstrap_interval": [ 0.12375511486331592, 0.13001807384582237 ], "confidence_level": 0.95, "mean": 0.12685049195699316, "replicates": 12, "sample_sd": 0.005791387311704008, "standard_error": 0.0016718295116968462 }, "log_loss": { "bootstrap_interval": [ 0.38910730398737614, 0.4053370008717838 ], "confidence_level": 0.95, "mean": 0.3972067695202899, "replicates": 12, "sample_sd": 0.015102281580766265, "standard_error": 0.004359653168016465 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.25, 0.3333333333333333, 0.5, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-5-5-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12276739215445667, -0.11928942687854197 ], "confidence_level": 0.95, "mean": -0.12100588076871815, "replicates": 12, "sample_sd": 0.003190811622065872, "standard_error": 0.0009211079744665589 }, "log_loss": { "bootstrap_interval": [ -0.37828274760957614, -0.36693843279594046 ], "confidence_level": 0.95, "mean": -0.3727523955794017, "replicates": 12, "sample_sd": 0.010359022654777365, "standard_error": 0.002990392259138572 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -5.934124441116818e-06, 6.632630382614807e-06 ], "confidence_level": 0.95, "mean": 6.651527652661028e-07, "replicates": 12, "sample_sd": 1.1017589977404444e-05, "standard_error": 3.1805042696376896e-06 }, "log_loss": { "bootstrap_interval": [ 1.0236031478905374e-05, 0.00016767519835406602 ], "confidence_level": 0.95, "mean": 7.134307261333465e-05, "replicates": 12, "sample_sd": 0.00015557896718026304, "standard_error": 4.491177929088441e-05 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -5.934124441118553e-06, 6.632630382614214e-06 ], "confidence_level": 0.95, "mean": 6.651527652649464e-07, "replicates": 12, "sample_sd": 1.101758997740452e-05, "standard_error": 3.1805042696377116e-06 }, "log_loss": { "bootstrap_interval": [ 1.023603147890656e-05, 0.00016767519835406722 ], "confidence_level": 0.95, "mean": 7.134307261333582e-05, "replicates": 12, "sample_sd": 0.00015557896718026247, "standard_error": 4.491177929088425e-05 } } }, "discount": { "bootstrap_interval": [ 0.9817711955085122, 0.9982977761558007 ], "confidence_level": 0.95, "mean": 0.9915007816867969, "replicates": 12, "sample_sd": 0.016032183180547065, "standard_error": 0.004628092637493119 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.9442049583543137, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16131052592176018, 0.16284057541426686 ], "confidence_level": 0.95, "mean": 0.1620594223786907, "replicates": 12, "sample_sd": 0.00140162669241573, "standard_error": 0.00040461477408479325 }, "log_loss": { "bootstrap_interval": [ 0.510842990103718, 0.513978402890556 ], "confidence_level": 0.95, "mean": 0.5124084443394333, "replicates": 12, "sample_sd": 0.0029536418712487857, "standard_error": 0.0008526429647276182 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.03900392209251055, 0.04326567959074533 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.1332512785376345, 0.14610341279377484 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.038976512719449984, 0.04324416387197747 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535432, "standard_error": 0.0011389171978370331 }, "log_loss": { "bootstrap_interval": [ 0.1332588652855061, 0.14618795747651425 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701015, "standard_error": 0.003474035026314449 } }, "independent": { "brier": { "bootstrap_interval": [ 0.038950062799593684, 0.04321384953788598 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.13328744081575092, 0.14647953639949293 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19705395869392217, 0.19993710132067283 ], "confidence_level": 0.95, "mean": 0.19855599037445573, "replicates": 12, "sample_sd": 0.002670294696841916, "standard_error": 0.0007708476810186552 }, "log_loss": { "bootstrap_interval": [ 0.5776981711924359, 0.5845956002551057 ], "confidence_level": 0.95, "mean": 0.5812378702037866, "replicates": 12, "sample_sd": 0.006406362602030368, "standard_error": 0.0018493575864042921 } }, "learned": { "brier": { "bootstrap_interval": [ 0.038928561751262754, 0.04321992542176419 ], "confidence_level": 0.95, "mean": 0.04105354160997255, "replicates": 12, "sample_sd": 0.003941385178711937, "standard_error": 0.0011377798969546692 }, "log_loss": { "bootstrap_interval": [ 0.13344326592219838, 0.14622299720961052 ], "confidence_level": 0.95, "mean": 0.13965604876003165, "replicates": 12, "sample_sd": 0.012022996752140452, "standard_error": 0.003470740205657143 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-5-5-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.04582392125061763, -0.04163035025551603 ], "confidence_level": 0.95, "mean": -0.04376268003413384, "replicates": 12, "sample_sd": 0.0038996842536957064, "standard_error": 0.0011257418768128805 }, "log_loss": { "bootstrap_interval": [ -0.13545863608674308, -0.12288264806918593 ], "confidence_level": 0.95, "mean": -0.12927820239482724, "replicates": 12, "sample_sd": 0.011575804632164639, "standard_error": 0.0033416469602333856 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -6.973134273531201e-06, 8.801033417876144e-05 ], "confidence_level": 0.95, "mean": 4.0026083755380824e-05, "replicates": 12, "sample_sd": 8.807813976674094e-05, "standard_error": 2.5425968852024684e-05 }, "log_loss": { "bootstrap_interval": [ 2.5605429502269006e-05, 0.00045146173365795953 ], "confidence_level": 0.95, "mean": 0.00022239800516709687, "replicates": 12, "sample_sd": 0.00039306209751537925, "standard_error": 0.00011346725390437157 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.020845048713318125, -0.017037243951255155 ], "confidence_level": 0.95, "mean": -0.018915133736768917, "replicates": 12, "sample_sd": 0.003517225063687474, "standard_error": 0.0010153354186602309 }, "log_loss": { "bootstrap_interval": [ -0.2368263530647193, -0.20521523466716116 ], "confidence_level": 0.95, "mean": -0.2209157370470596, "replicates": 12, "sample_sd": 0.029528724492976072, "standard_error": 0.008524208517423016 } } }, "discount": { "bootstrap_interval": [ 0.32896457742554996, 0.3385551223441874 ], "confidence_level": 0.95, "mean": 0.33368797518965165, "replicates": 12, "sample_sd": 0.00883523613573111, "standard_error": 0.002550512980659133 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.3483039547172565, "discount_min": 0.3207934762683671, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16827176490274157, 0.1714299923779983 ], "confidence_level": 0.95, "mean": 0.16984463263408034, "replicates": 12, "sample_sd": 0.002939932133692854, "standard_error": 0.0008486853043934001 }, "log_loss": { "bootstrap_interval": [ 0.5208144163903374, 0.5275652247272054 ], "confidence_level": 0.95, "mean": 0.5241468636767158, "replicates": 12, "sample_sd": 0.006224699165038559, "standard_error": 0.0017969158692797255 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1335310605117262, 0.13874762186904896 ], "confidence_level": 0.95, "mean": 0.13611428670851633, "replicates": 12, "sample_sd": 0.0047874785938498594, "standard_error": 0.0013820260274493938 }, "log_loss": { "bootstrap_interval": [ 0.4271452707817712, 0.43894777923707134 ], "confidence_level": 0.95, "mean": 0.43291588387774105, "replicates": 12, "sample_sd": 0.0109690216569211, "standard_error": 0.003166483803185116 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.12241870488336055, 0.12956226196174894 ], "confidence_level": 0.95, "mean": 0.12604192651619112, "replicates": 12, "sample_sd": 0.006675153372323958, "standard_error": 0.0019269507981966377 }, "log_loss": { "bootstrap_interval": [ 0.38545044329541434, 0.404205654214409 ], "confidence_level": 0.95, "mean": 0.3946462632767214, "replicates": 12, "sample_sd": 0.017418747787293683, "standard_error": 0.005028359361970104 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1397292153233236, 0.15035426713548927 ], "confidence_level": 0.95, "mean": 0.14499708633671543, "replicates": 12, "sample_sd": 0.00985032811626449, "standard_error": 0.0028435447947657215 }, "log_loss": { "bootstrap_interval": [ 0.5904180606001479, 0.6413671924721925 ], "confidence_level": 0.95, "mean": 0.6157843983289482, "replicates": 12, "sample_sd": 0.04639387267733929, "standard_error": 0.013392757439505533 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19710828814879489, 0.20212778925036293 ], "confidence_level": 0.95, "mean": 0.19956596318459022, "replicates": 12, "sample_sd": 0.004569486908080526, "standard_error": 0.0013190972482193812 }, "log_loss": { "bootstrap_interval": [ 0.5776308125270172, 0.5890591485117616 ], "confidence_level": 0.95, "mean": 0.5832302241109455, "replicates": 12, "sample_sd": 0.010553555878235293, "standard_error": 0.003046549163603452 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1224589156759085, 0.12968066875907103 ], "confidence_level": 0.95, "mean": 0.1260819525999465, "replicates": 12, "sample_sd": 0.006687026385968284, "standard_error": 0.0019303782420084597 }, "log_loss": { "bootstrap_interval": [ 0.3855511665863488, 0.4043557916110764 ], "confidence_level": 0.95, "mean": 0.3948686612818885, "replicates": 12, "sample_sd": 0.01749253316805112, "standard_error": 0.005049659366691385 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.3333333333333333, 0.3333333333333333 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.3333333333333333, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-5-5-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.021263596252041425, -0.01794579769292175 ], "confidence_level": 0.95, "mean": -0.01956107622251899, "replicates": 12, "sample_sd": 0.0030762693589608883, "standard_error": 0.0008880424712479333 }, "log_loss": { "bootstrap_interval": [ -0.0628277841588091, -0.05291081655451678 ], "confidence_level": 0.95, "mean": -0.058002179265145205, "replicates": 12, "sample_sd": 0.009202420998149894, "standard_error": 0.002656510120239053 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -3.5216407158670145e-05, 9.31070901746736e-05 ], "confidence_level": 0.95, "mean": 2.709040592966533e-05, "replicates": 12, "sample_sd": 0.00011638598951650064, "standard_error": 3.3597741188626306e-05 }, "log_loss": { "bootstrap_interval": [ -6.943425566249433e-05, 0.0004136573493801077 ], "confidence_level": 0.95, "mean": 0.00015423649418323804, "replicates": 12, "sample_sd": 0.0004520531104837149, "standard_error": 0.00013049649251289022 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.03899801845133998, -0.03366429220834236 ], "confidence_level": 0.95, "mean": -0.03649497497718091, "replicates": 12, "sample_sd": 0.004942635166236391, "standard_error": 0.0014268158718663455 }, "log_loss": { "bootstrap_interval": [ -0.5350568040321136, -0.47957093720643 ], "confidence_level": 0.95, "mean": -0.5072567651143154, "replicates": 12, "sample_sd": 0.051559078825058856, "standard_error": 0.014883824019408434 } } }, "discount": { "bootstrap_interval": [ 0.2147914833624303, 0.22158433383523682 ], "confidence_level": 0.95, "mean": 0.2182660154815549, "replicates": 12, "sample_sd": 0.0062790035714688375, "standard_error": 0.0018125922011150775 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.2280043886926027, "discount_min": 0.20591437398083684, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17451830322876072, 0.1784914063263694 ], "confidence_level": 0.95, "mean": 0.17652239506866968, "replicates": 12, "sample_sd": 0.00368210348541881, "standard_error": 0.0010629317192453046 }, "log_loss": { "bootstrap_interval": [ 0.531057039667704, 0.5397302595597204 ], "confidence_level": 0.95, "mean": 0.5353402132712943, "replicates": 12, "sample_sd": 0.007918816487382737, "standard_error": 0.002285965415326835 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.16684985503024116, 0.17142993515781052 ], "confidence_level": 0.95, "mean": 0.16911283399555654, "replicates": 12, "sample_sd": 0.004283156759779427, "standard_error": 0.0012364408541200088 }, "log_loss": { "bootstrap_interval": [ 0.5110776493401916, 0.5212685849374721 ], "confidence_level": 0.95, "mean": 0.5160590683274893, "replicates": 12, "sample_sd": 0.009420865817867849, "standard_error": 0.0027195697079726733 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15342538609389342, 0.16046184389967488 ], "confidence_level": 0.95, "mean": 0.156934228440221, "replicates": 12, "sample_sd": 0.006585552766888142, "standard_error": 0.001901085331362677 }, "log_loss": { "bootstrap_interval": [ 0.468174055336278, 0.4861789949732009 ], "confidence_level": 0.95, "mean": 0.4771837975119658, "replicates": 12, "sample_sd": 0.0166645631271402, "standard_error": 0.0048106450036909535 } }, "independent": { "brier": { "bootstrap_interval": [ 0.187449069477209, 0.19892357198445018 ], "confidence_level": 0.95, "mean": 0.1934562938233316, "replicates": 12, "sample_sd": 0.010665024554659907, "standard_error": 0.003078727398773433 }, "log_loss": { "bootstrap_interval": [ 0.9475366445102411, 1.0205238408633226 ], "confidence_level": 0.95, "mean": 0.9845947991204644, "replicates": 12, "sample_sd": 0.06772457965500137, "standard_error": 0.019550402147284648 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19673950935805531, 0.20323679984921994 ], "confidence_level": 0.95, "mean": 0.1999359132791692, "replicates": 12, "sample_sd": 0.005988078724364737, "standard_error": 0.0017286094317203258 }, "log_loss": { "bootstrap_interval": [ 0.5767072643673584, 0.5917799664480259 ], "confidence_level": 0.95, "mean": 0.5842178182554794, "replicates": 12, "sample_sd": 0.013890520585885505, "standard_error": 0.004009847899722517 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15328162583997396, 0.16046802598741894 ], "confidence_level": 0.95, "mean": 0.1569613188461507, "replicates": 12, "sample_sd": 0.006642571291907098, "standard_error": 0.0019175451617469217 }, "log_loss": { "bootstrap_interval": [ 0.4683384339863092, 0.4864541030598563 ], "confidence_level": 0.95, "mean": 0.4773380340061491, "replicates": 12, "sample_sd": 0.016888541735157078, "standard_error": 0.004875302058506585 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2173913043478261, 0.2173913043478261 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-5-5-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.017257685120512907, -0.014283865717793706 ], "confidence_level": 0.95, "mean": -0.015687892611242458, "replicates": 12, "sample_sd": 0.0027969646378351613, "standard_error": 0.0008074141432839973 }, "log_loss": { "bootstrap_interval": [ -0.05155056534769676, -0.042373005731378914 ], "confidence_level": 0.95, "mean": -0.046863025306333385, "replicates": 12, "sample_sd": 0.008386392483275287, "standard_error": 0.002420942978874421 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -4.730876057456404e-05, 8.24706467512642e-05 ], "confidence_level": 0.95, "mean": 1.6838939999333502e-05, "replicates": 12, "sample_sd": 0.0001215541544917744, "standard_error": 3.508966190847166e-05 }, "log_loss": { "bootstrap_interval": [ -0.00010972019467151011, 0.000375441261021541 ], "confidence_level": 0.95, "mean": 0.00012212219672716, "replicates": 12, "sample_sd": 0.00044281100241316975, "standard_error": 0.0001278285257216858 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04390852461126875, -0.03829501388463667 ], "confidence_level": 0.95, "mean": -0.04108195273601979, "replicates": 12, "sample_sd": 0.005145825507609817, "standard_error": 0.0014854718710106852 }, "log_loss": { "bootstrap_interval": [ -0.6110377123796404, -0.5510673515254784 ], "confidence_level": 0.95, "mean": -0.5816340477236612, "replicates": 12, "sample_sd": 0.054844082280955414, "standard_error": 0.0158321228341838 } } }, "discount": { "bootstrap_interval": [ 0.19784411806480906, 0.2039949590048865 ], "confidence_level": 0.95, "mean": 0.20102649413852158, "replicates": 12, "sample_sd": 0.005730512618345523, "standard_error": 0.0016542565013981676 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.20902682366682912, "discount_min": 0.18914190366712574, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 5, 5 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17633543584594258, 0.18041994201734415 ], "confidence_level": 0.95, "mean": 0.17834674818486793, "replicates": 12, "sample_sd": 0.0037708399616126266, "standard_error": 0.0010885477334540241 }, "log_loss": { "bootstrap_interval": [ 0.5341942471752245, 0.5429807455730625 ], "confidence_level": 0.95, "mean": 0.5385617132887028, "replicates": 12, "sample_sd": 0.008173672850337675, "standard_error": 0.0023595361102051963 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.17256109381740098, 0.17696213977452172 ], "confidence_level": 0.95, "mean": 0.1747625663833307, "replicates": 12, "sample_sd": 0.004067146603119908, "standard_error": 0.0011740840930724754 }, "log_loss": { "bootstrap_interval": [ 0.5246333471877853, 0.5343306686511626 ], "confidence_level": 0.95, "mean": 0.529476656666323, "replicates": 12, "sample_sd": 0.008918444585019778, "standard_error": 0.002574533190956965 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15910788225315167, 0.1660418560732499 ], "confidence_level": 0.95, "mean": 0.16264201663362615, "replicates": 12, "sample_sd": 0.0063380316224901515, "standard_error": 0.0018296321316885248 }, "log_loss": { "bootstrap_interval": [ 0.48285250187035283, 0.5001016010832182 ], "confidence_level": 0.95, "mean": 0.49157656578564235, "replicates": 12, "sample_sd": 0.016012033170783823, "standard_error": 0.004622275830712629 } }, "independent": { "brier": { "bootstrap_interval": [ 0.19807630127213047, 0.20889442606127331 ], "confidence_level": 0.95, "mean": 0.20374080830964528, "replicates": 12, "sample_sd": 0.010070657014400351, "standard_error": 0.002907148269090218 }, "log_loss": { "bootstrap_interval": [ 1.034634792802494, 1.1109165040939863 ], "confidence_level": 0.95, "mean": 1.0733327357060307, "replicates": 12, "sample_sd": 0.07021658313838251, "standard_error": 0.02026978158826044 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19669746244063593, 0.20353838255173803 ], "confidence_level": 0.95, "mean": 0.2000848566280008, "replicates": 12, "sample_sd": 0.006311011877348332, "standard_error": 0.0018218322031229926 }, "log_loss": { "bootstrap_interval": [ 0.5766833299397442, 0.5926116368777231 ], "confidence_level": 0.95, "mean": 0.5844790344204306, "replicates": 12, "sample_sd": 0.01460589757934671, "standard_error": 0.004216359449595964 } }, "learned": { "brier": { "bootstrap_interval": [ 0.15921627694968427, 0.16612257045245732 ], "confidence_level": 0.95, "mean": 0.16265885557362547, "replicates": 12, "sample_sd": 0.006413123734596294, "standard_error": 0.0018513093572577743 }, "log_loss": { "bootstrap_interval": [ 0.48282943667181805, 0.5004604511187486 ], "confidence_level": 0.95, "mean": 0.4916986879823695, "replicates": 12, "sample_sd": 0.01627658435374354, "standard_error": 0.004698645179060742 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 0.2 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.2, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-9-1-rho0": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.12276195257142072, -0.11928082864846133 ], "confidence_level": 0.95, "mean": -0.12100588076871815, "replicates": 12, "sample_sd": 0.003190811622065872, "standard_error": 0.0009211079744665589 }, "log_loss": { "bootstrap_interval": [ -0.3781648237227772, -0.36705688581284657 ], "confidence_level": 0.95, "mean": -0.3727523955794017, "replicates": 12, "sample_sd": 0.010359022654777365, "standard_error": 0.002990392259138572 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ -5.92183118673247e-06, 6.450058452751757e-06 ], "confidence_level": 0.95, "mean": 6.651527652655246e-07, "replicates": 12, "sample_sd": 1.1017589977404482e-05, "standard_error": 3.1805042696377e-06 }, "log_loss": { "bootstrap_interval": [ 1.1262484632632946e-05, 0.00016798653931606133 ], "confidence_level": 0.95, "mean": 7.134307261333582e-05, "replicates": 12, "sample_sd": 0.00015557896718026247, "standard_error": 4.491177929088425e-05 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -5.921831186733641e-06, 6.450058452751757e-06 ], "confidence_level": 0.95, "mean": 6.651527652649464e-07, "replicates": 12, "sample_sd": 1.101758997740452e-05, "standard_error": 3.1805042696377116e-06 }, "log_loss": { "bootstrap_interval": [ 1.1262484632632946e-05, 0.00016798653931606133 ], "confidence_level": 0.95, "mean": 7.134307261333582e-05, "replicates": 12, "sample_sd": 0.00015557896718026247, "standard_error": 4.491177929088425e-05 } } }, "discount": { "bootstrap_interval": [ 0.981786783811209, 0.9981962711948962 ], "confidence_level": 0.95, "mean": 0.9915007816867969, "replicates": 12, "sample_sd": 0.016032183180547065, "standard_error": 0.004628092637493119 }, "discount_below_exchangeable_range_count": 0, "discount_max": 1.0, "discount_min": 0.9442049583543137, "discount_one_count": 6, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.16131746905162597, 0.162817435331973 ], "confidence_level": 0.95, "mean": 0.1620594223786907, "replicates": 12, "sample_sd": 0.00140162669241573, "standard_error": 0.00040461477408479325 }, "log_loss": { "bootstrap_interval": [ 0.510816415008216, 0.5139948841265018 ], "confidence_level": 0.95, "mean": 0.5124084443394333, "replicates": 12, "sample_sd": 0.0029536418712487857, "standard_error": 0.0008526429647276182 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.03899734616441753, 0.04325304571836103 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.13349632749430404, 0.14618865312567111 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.038970692577157226, 0.04320604953015819 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.13326036638800506, 0.14614043013355085 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "independent": { "brier": { "bootstrap_interval": [ 0.038963776359052286, 0.04326332388200503 ], "confidence_level": 0.95, "mean": 0.04105287645720729, "replicates": 12, "sample_sd": 0.003945324904535433, "standard_error": 0.0011389171978370334 }, "log_loss": { "bootstrap_interval": [ 0.13318903829751844, 0.14608174408314126 ], "confidence_level": 0.95, "mean": 0.13958470568741832, "replicates": 12, "sample_sd": 0.012034410345701017, "standard_error": 0.00347403502631445 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19705972393790722, 0.1999156927479105 ], "confidence_level": 0.95, "mean": 0.19855599037445573, "replicates": 12, "sample_sd": 0.002670294696841916, "standard_error": 0.0007708476810186552 }, "log_loss": { "bootstrap_interval": [ 0.5775671617173, 0.5844771429607309 ], "confidence_level": 0.95, "mean": 0.5812378702037866, "replicates": 12, "sample_sd": 0.006406362602030368, "standard_error": 0.0018493575864042921 } }, "learned": { "brier": { "bootstrap_interval": [ 0.038964914494800836, 0.043213111121754424 ], "confidence_level": 0.95, "mean": 0.04105354160997255, "replicates": 12, "sample_sd": 0.003941385178711937, "standard_error": 0.0011377798969546692 }, "log_loss": { "bootstrap_interval": [ 0.1332326188969824, 0.14621425607471175 ], "confidence_level": 0.95, "mean": 0.13965604876003165, "replicates": 12, "sample_sd": 0.012022996752140452, "standard_error": 0.003470740205657143 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 1.0, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 1.0, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": true, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-9-1-rho0.5": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.021057853974512212, -0.01729264818025686 ], "confidence_level": 0.95, "mean": -0.01919768217979454, "replicates": 12, "sample_sd": 0.003519376076512904, "standard_error": 0.0010159563625771271 }, "log_loss": { "bootstrap_interval": [ -0.06191044214104792, -0.0511119468476166 ], "confidence_level": 0.95, "mean": -0.05657260410391448, "replicates": 12, "sample_sd": 0.010028396654218329, "standard_error": 0.0028949487539266473 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.017468537714970075, 0.020218754506276467 ], "confidence_level": 0.95, "mean": 0.018833360226735707, "replicates": 12, "sample_sd": 0.0025437403481519093, "standard_error": 0.0007343145873770086 }, "log_loss": { "bootstrap_interval": [ 0.047259483815486586, 0.05412101049299372 ], "confidence_level": 0.95, "mean": 0.050560736649579295, "replicates": 12, "sample_sd": 0.0064058277204792746, "standard_error": 0.001849203179400538 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.04094123846936641, -0.034462525981739094 ], "confidence_level": 0.95, "mean": -0.03780319595863569, "replicates": 12, "sample_sd": 0.0059519268154289715, "standard_error": 0.0017181732745424345 }, "log_loss": { "bootstrap_interval": [ -0.5502087970849951, -0.48774894369525024 ], "confidence_level": 0.95, "mean": -0.5189189903565979, "replicates": 12, "sample_sd": 0.05813945393076882, "standard_error": 0.016783414688733612 } } }, "discount": { "bootstrap_interval": [ 0.21528813393792684, 0.22101443193731268 ], "confidence_level": 0.95, "mean": 0.21816681560235535, "replicates": 12, "sample_sd": 0.005283339859864942, "standard_error": 0.001525168845156652 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.22768003596234648, "discount_min": 0.2099464219119273, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.17494236138683755, 0.1792811525460794 ], "confidence_level": 0.95, "mean": 0.1770619640407245, "replicates": 12, "sample_sd": 0.0041037557277600265, "standard_error": 0.00118465223705536 }, "log_loss": { "bootstrap_interval": [ 0.5320648194895994, 0.5415699763704378 ], "confidence_level": 0.95, "mean": 0.5367291366961829, "replicates": 12, "sample_sd": 0.008780936194294322, "standard_error": 0.0025348379377563776 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1551168889835765, 0.1623732265184045 ], "confidence_level": 0.95, "mean": 0.158659884747626, "replicates": 12, "sample_sd": 0.00669801861778189, "standard_error": 0.0019335514260067498 }, "log_loss": { "bootstrap_interval": [ 0.47463509893928235, 0.49190984117133024 ], "confidence_level": 0.95, "mean": 0.4831811096797788, "replicates": 12, "sample_sd": 0.015817226586386034, "standard_error": 0.004566040013741641 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.13496964644605555, 0.14309030866712752 ], "confidence_level": 0.95, "mean": 0.13903092163419425, "replicates": 12, "sample_sd": 0.007395678980536409, "standard_error": 0.0021349486251263767 }, "log_loss": { "bootstrap_interval": [ 0.41895963861966884, 0.440405840431055 ], "confidence_level": 0.95, "mean": 0.42959579594268915, "replicates": 12, "sample_sd": 0.019683353129160086, "standard_error": 0.005682094613837519 } }, "independent": { "brier": { "bootstrap_interval": [ 0.1887483581786456, 0.2025627326059191 ], "confidence_level": 0.95, "mean": 0.19566747781956564, "replicates": 12, "sample_sd": 0.012742893466860295, "standard_error": 0.0036785564866732576 }, "log_loss": { "bootstrap_interval": [ 0.9582365192837482, 1.0406584900064828 ], "confidence_level": 0.95, "mean": 0.9990755229488664, "replicates": 12, "sample_sd": 0.07641580873341675, "standard_error": 0.02205934387129056 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19717046300966623, 0.20207755462843244 ], "confidence_level": 0.95, "mean": 0.19956596318459022, "replicates": 12, "sample_sd": 0.004569486908080526, "standard_error": 0.0013190972482193812 }, "log_loss": { "bootstrap_interval": [ 0.5777316567313744, 0.5892663861891817 ], "confidence_level": 0.95, "mean": 0.5832302241109455, "replicates": 12, "sample_sd": 0.010553555878235293, "standard_error": 0.003046549163603452 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1538710333420122, 0.16197135305668375 ], "confidence_level": 0.95, "mean": 0.15786428186092996, "replicates": 12, "sample_sd": 0.007494626615255645, "standard_error": 0.00216351234689679 }, "log_loss": { "bootstrap_interval": [ 0.47049742090821794, 0.4903292441033917 ], "confidence_level": 0.95, "mean": 0.4801565325922684, "replicates": 12, "sample_sd": 0.01857168337245582, "standard_error": 0.0053611831971959335 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.2, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.21739130434782608, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-9-1-rho0.9": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0038266418763667908, -0.0020087153496168653 ], "confidence_level": 0.95, "mean": -0.002886892169433342, "replicates": 12, "sample_sd": 0.0016783358615428694, "standard_error": 0.0004844938307261891 }, "log_loss": { "bootstrap_interval": [ -0.010801643773381714, -0.005614282110862172 ], "confidence_level": 0.95, "mean": -0.008215809535452359, "replicates": 12, "sample_sd": 0.004826094659752323, "standard_error": 0.0013931735254713094 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.026047908538054533, 0.028725938006382724 ], "confidence_level": 0.95, "mean": 0.027374680177290302, "replicates": 12, "sample_sd": 0.002486136431160268, "standard_error": 0.0007176857688862582 }, "log_loss": { "bootstrap_interval": [ 0.06633503332364127, 0.07290938834147609 ], "confidence_level": 0.95, "mean": 0.06957665014731274, "replicates": 12, "sample_sd": 0.0061352908753032715, "standard_error": 0.0017711059192064994 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.06622762592688877, -0.05790581683823025 ], "confidence_level": 0.95, "mean": -0.06217446456929805, "replicates": 12, "sample_sd": 0.007777381906987997, "standard_error": 0.0022451367687950224 }, "log_loss": { "bootstrap_interval": [ -1.1067820119197154, -0.9934580258249719 ], "confidence_level": 0.95, "mean": -1.0502748324131201, "replicates": 12, "sample_sd": 0.1055884428432916, "standard_error": 0.03048075794944391 } } }, "discount": { "bootstrap_interval": [ 0.13252763235792053, 0.13646193195857362 ], "confidence_level": 0.95, "mean": 0.13449305263725966, "replicates": 12, "sample_sd": 0.0036738270484991276, "standard_error": 0.00106054251770355 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.14153079939701851, "discount_min": 0.1290536892882424, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18636559576593464, 0.1921844647450813 ], "confidence_level": 0.95, "mean": 0.1892973739344224, "replicates": 12, "sample_sd": 0.005499058310938704, "standard_error": 0.0015874413980549549 }, "log_loss": { "bootstrap_interval": [ 0.5536540487415006, 0.566630869721707 ], "confidence_level": 0.95, "mean": 0.5600877542854446, "replicates": 12, "sample_sd": 0.012099923981259275, "standard_error": 0.0034929471838770257 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.1840255031134429, 0.1906631597421492 ], "confidence_level": 0.95, "mean": 0.18728758826693237, "replicates": 12, "sample_sd": 0.006149343936053933, "standard_error": 0.0017751626884101657 }, "log_loss": { "bootstrap_interval": [ 0.5470660286987978, 0.5619875843909046 ], "confidence_level": 0.95, "mean": 0.5545729553250672, "replicates": 12, "sample_sd": 0.013882820227329284, "standard_error": 0.004007624997679872 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15537789159680238, 0.16270004031215388 ], "confidence_level": 0.95, "mean": 0.15903580158769876, "replicates": 12, "sample_sd": 0.006953487792557899, "standard_error": 0.00200729902442004 }, "log_loss": { "bootstrap_interval": [ 0.47273861577286297, 0.49199423169889156 ], "confidence_level": 0.95, "mean": 0.4822952946026794, "replicates": 12, "sample_sd": 0.017739168493674757, "standard_error": 0.005120856852511625 } }, "independent": { "brier": { "bootstrap_interval": [ 0.24073145717446878, 0.2563756476403126 ], "confidence_level": 0.95, "mean": 0.2485849463342871, "replicates": 12, "sample_sd": 0.014541665334178968, "standard_error": 0.004197817197576838 }, "log_loss": { "bootstrap_interval": [ 1.5346784169363594, 1.6680493398153209 ], "confidence_level": 0.95, "mean": 1.6021467771631124, "replicates": 12, "sample_sd": 0.12211833669459744, "standard_error": 0.0352525272818076 } }, "individual": { "brier": { "bootstrap_interval": [ 0.19670768327030091, 0.20325498267756603 ], "confidence_level": 0.95, "mean": 0.1999359132791692, "replicates": 12, "sample_sd": 0.005988078724364737, "standard_error": 0.0017286094317203258 }, "log_loss": { "bootstrap_interval": [ 0.5767880468234541, 0.591898266371523 ], "confidence_level": 0.95, "mean": 0.5842178182554794, "replicates": 12, "sample_sd": 0.013890520585885505, "standard_error": 0.004009847899722517 } }, "learned": { "brier": { "bootstrap_interval": [ 0.1826354246612981, 0.19028302464590272 ], "confidence_level": 0.95, "mean": 0.18641048176498906, "replicates": 12, "sample_sd": 0.0071422255404134365, "standard_error": 0.002061782919185359 }, "log_loss": { "bootstrap_interval": [ 0.5427780580647353, 0.5609526254851427 ], "confidence_level": 0.95, "mean": 0.5518719447499921, "replicates": 12, "sample_sd": 0.016835108480370512, "standard_error": 0.004859877206489234 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.12195121951219513, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.13368983957219252, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 }, "groups-9-1-rho1": { "condition": { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, "contrasts": { "learned_minus_arithmetic_mean": { "brier": { "bootstrap_interval": [ -0.0020565181797282304, -0.0007529302903472245 ], "confidence_level": 0.95, "mean": -0.001382321955512004, "replicates": 12, "sample_sd": 0.0012019962776634403, "standard_error": 0.0003469864372369577 }, "log_loss": { "bootstrap_interval": [ -0.005437602971041474, -0.001602878693787094 ], "confidence_level": 0.95, "mean": -0.003544412602817709, "replicates": 12, "sample_sd": 0.003570494844047782, "standard_error": 0.001030713079675579 } }, "learned_minus_grouped_oracle": { "brier": { "bootstrap_interval": [ 0.027261433853889835, 0.029685077765598273 ], "confidence_level": 0.95, "mean": 0.028466002125436336, "replicates": 12, "sample_sd": 0.002203735652445135, "standard_error": 0.0006361636860809871 }, "log_loss": { "bootstrap_interval": [ 0.06864297259198827, 0.0745436157713759 ], "confidence_level": 0.95, "mean": 0.07157177497126342, "replicates": 12, "sample_sd": 0.00541195171828683, "standard_error": 0.0015622958906970797 } }, "learned_minus_independent": { "brier": { "bootstrap_interval": [ -0.07138085304845397, -0.06283119568636135 ], "confidence_level": 0.95, "mean": -0.06717479586047675, "replicates": 12, "sample_sd": 0.007799682322601297, "standard_error": 0.002251574344273712 }, "log_loss": { "bootstrap_interval": [ -1.2407642404970955, -1.115830706212713 ], "confidence_level": 0.95, "mean": -1.1783486666377114, "replicates": 12, "sample_sd": 0.11618859673385706, "standard_error": 0.033540758800528626 } } }, "discount": { "bootstrap_interval": [ 0.12093816047381988, 0.12452138405248352 ], "confidence_level": 0.95, "mean": 0.12272321245004311, "replicates": 12, "sample_sd": 0.0033691008466137256, "standard_error": 0.0009725756403597153 }, "discount_below_exchangeable_range_count": 0, "discount_max": 0.12902434450893369, "discount_min": 0.11813477352758095, "discount_one_count": 0, "discount_zero_count": 0, "group_sizes": [ 9, 1 ], "methods": { "arithmetic_mean": { "brier": { "bootstrap_interval": [ 0.18942403709009178, 0.19559103728214808 ], "confidence_level": 0.95, "mean": 0.19249034071457446, "replicates": 12, "sample_sd": 0.005727569130674028, "standard_error": 0.0016534067896984206 }, "log_loss": { "bootstrap_interval": [ 0.5598780800065424, 0.5735332872520241 ], "confidence_level": 0.95, "mean": 0.5666927533597235, "replicates": 12, "sample_sd": 0.012734177263388666, "standard_error": 0.0036760403354629293 } }, "exchangeable_formula": { "brier": { "bootstrap_interval": [ 0.18876381539410358, 0.19518697109194552 ], "confidence_level": 0.95, "mean": 0.19194534648622238, "replicates": 12, "sample_sd": 0.0059107957747690535, "standard_error": 0.001706299765843908 }, "log_loss": { "bootstrap_interval": [ 0.5586262016052391, 0.5729999740713949 ], "confidence_level": 0.95, "mean": 0.5656691592356481, "replicates": 12, "sample_sd": 0.013294900058021064, "standard_error": 0.0038379070636738164 } }, "grouped_oracle": { "brier": { "bootstrap_interval": [ 0.15914411530902398, 0.16596411365470065 ], "confidence_level": 0.95, "mean": 0.16264201663362615, "replicates": 12, "sample_sd": 0.0063380316224901515, "standard_error": 0.0018296321316885248 }, "log_loss": { "bootstrap_interval": [ 0.4830852427829154, 0.5002396964302671 ], "confidence_level": 0.95, "mean": 0.49157656578564235, "replicates": 12, "sample_sd": 0.016012033170783823, "standard_error": 0.004622275830712629 } }, "independent": { "brier": { "bootstrap_interval": [ 0.2504890766434629, 0.2659420044222523 ], "confidence_level": 0.95, "mean": 0.2582828146195392, "replicates": 12, "sample_sd": 0.014330023806473283, "standard_error": 0.004136721551080548 }, "log_loss": { "bootstrap_interval": [ 1.670126613030092, 1.8122095653189476 ], "confidence_level": 0.95, "mean": 1.7414970073946172, "replicates": 12, "sample_sd": 0.13199225166023862, "standard_error": 0.038102881013491796 } }, "individual": { "brier": { "bootstrap_interval": [ 0.1965447010320893, 0.20352901160173076 ], "confidence_level": 0.95, "mean": 0.2000848566280008, "replicates": 12, "sample_sd": 0.006311011877348332, "standard_error": 0.0018218322031229926 }, "log_loss": { "bootstrap_interval": [ 0.5766665906623231, 0.5923230511246407 ], "confidence_level": 0.95, "mean": 0.5844790344204306, "replicates": 12, "sample_sd": 0.01460589757934671, "standard_error": 0.004216359449595964 } }, "learned": { "brier": { "bootstrap_interval": [ 0.18735323875168322, 0.1948324540780573 ], "confidence_level": 0.95, "mean": 0.19110801875906247, "replicates": 12, "sample_sd": 0.006901884344606111, "standard_error": 0.001992402392137001 }, "log_loss": { "bootstrap_interval": [ 0.5546945910813436, 0.5716979288835745 ], "confidence_level": 0.95, "mean": 0.5631483407569058, "replicates": 12, "sample_sd": 0.016191686498950773, "standard_error": 0.0046741372794016295 } }, "prior": { "brier": { "bootstrap_interval": [ 0.25, 0.25 ], "confidence_level": 0.95, "mean": 0.25, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 }, "log_loss": { "bootstrap_interval": [ 0.6931471805599453, 0.6931471805599453 ], "confidence_level": 0.95, "mean": 0.6931471805599453, "replicates": 12, "sample_sd": 0.0, "standard_error": 0.0 } } }, "oracle_group_weights": [ 0.1111111111111111, 1.0 ], "oracle_method": "grouped_oracle", "population_optimal_scalar": 0.12195121951219512, "population_scalar_identified": true, "replicate_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], "scalar_can_represent_oracle": false, "test_count_per_replicate": 2000, "train_count_per_replicate": 5000 } }, "config": { "conditions": [ { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "n_forecasters": 10, "name": "groups-5-5-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 ], "n_forecasters": 10, "name": "groups-9-1-rho1", "overlap": 1, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0", "overlap": 0, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.5", "overlap": 0.5, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho0.9", "overlap": 0.9, "signal_strength": 0.5 }, { "group_ids": [ 0, 0, 0, 0, 1, 1, 1, 2, 2, 3 ], "n_forecasters": 10, "name": "groups-4-3-2-1-rho1", "overlap": 1, "signal_strength": 0.5 } ], "phase": "exploratory-stress-groups-m5000-v1", "replicates": 12, "schema_version": 1, "seed": 84173103, "test_count": 2000, "train_count": 5000 }, "config_sha256": "503541d99d098122fa6957a81106a76a5430a7a890150e3b78126f6710863643", "expected_runs": 144, "input_path": "results/stress-v1/stress-groups-m5000-v1.json", "input_sha256": "76c06f1183084ea86bdc6bcfce2654804c68b81a300eade33e668f913dfe357a", "ledger_receipt": { "bytes": 162671436, "count": 1008000, "payload_sha256": "ef36c4e97d7e2fac53d6288e25d9af984155299f6d1202eb6c5fd988f5b6aadd", "schema_version": 1, "sha256": "47b08813cbfb358b354460b3557937779e50709749c4f704fea500c66f5ec267" }, "manifest_sha256": null, "observed_runs": 144, "phase": "exploratory-stress-groups-m5000-v1", "source_sha256": { "archive.py": "2f7138bec37be77258917267ad905ea70297460c04b01f4248716730dd869df6", "evaluate.py": "d9ac3c35763eb6a0aebf17fd0ab249287a57b73888cc6b1f2d7b0b36e135f7fb", "experiment.py": "c65d1f2d06d902eb16f442d5f4ee23db959006981462e6775baf3dcbbbe9df68", "groups.py": "99d2765941f113fd47cc15861d4fc975bcb80768f589b677df1a6ab13a7cc5a6", "ledger.py": "5e066bc506c2c5f9a1f4f0ee13a639d305440822979c97246c4fd30238d3da19", "model.py": "7cdb2a2702c46599ba90f659667787894dd2d53d17cf170371ec3ab594720674", "simulate.py": "eb395735b9468d8901de5d02f04db48d5377df69dfd65ba5a1fce06a060f60bf" }, "status": "development_diagnostic" } }, "grouped_opposite_sign_diagnostic": "Not implemented in this report; requires a separate matching-ledger diagnostic.", "interpretation": "Every input cell retained. Intervals are descriptive, not adjusted across cells. Conditions can share random draws; no pooled cross-cell inference is made. Negative realized oracle regret can occur from test sampling.", "schema_version": 1, "status": "descriptive_synthetic_stress_report", "stress_lock_sha256": "25c5f13ec08251d091d14dc12f394885a3e3653046e01c37b7c761e1871455b0", "verification_scope": "Summary schema, cells, score domains, and fitted bounds checked; ledger replay is separate." }