{
  "aggregate_results": {
    "attention": {
      "by_source_slice": {
        "bbh-logical-deduction-five-objects": {
          "accuracy": 0.376,
          "correct": 94,
          "examples": 250,
          "multiclass_brier_score": 0.6960678132755836,
          "negative_log_likelihood": 1.3693089668150822,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-logical-deduction-seven-objects": {
          "accuracy": 0.4,
          "correct": 100,
          "examples": 250,
          "multiclass_brier_score": 0.7128170759769269,
          "negative_log_likelihood": 1.5685680380749016,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-logical-deduction-three-objects": {
          "accuracy": 0.416,
          "correct": 104,
          "examples": 250,
          "multiclass_brier_score": 0.6252822327966882,
          "negative_log_likelihood": 1.0040964467232076,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "bbh-tracking-shuffled-objects-five-objects": {
          "accuracy": 0.18,
          "correct": 45,
          "examples": 250,
          "multiclass_brier_score": 0.9430917676234645,
          "negative_log_likelihood": 2.022383091205189,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-tracking-shuffled-objects-seven-objects": {
          "accuracy": 0.156,
          "correct": 39,
          "examples": 250,
          "multiclass_brier_score": 1.0149567161270374,
          "negative_log_likelihood": 2.491187762878268,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-tracking-shuffled-objects-three-objects": {
          "accuracy": 0.296,
          "correct": 74,
          "examples": 250,
          "multiclass_brier_score": 0.8592194109675687,
          "negative_log_likelihood": 1.4968598401964195,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "belebele-deu_Latn": {
          "accuracy": 0.5733333333333334,
          "correct": 516,
          "examples": 900,
          "multiclass_brier_score": 0.5557755894384145,
          "negative_log_likelihood": 1.0763926787894271,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-eng_Latn": {
          "accuracy": 0.5888888888888889,
          "correct": 530,
          "examples": 900,
          "multiclass_brier_score": 0.5675780702457816,
          "negative_log_likelihood": 1.1676345363319927,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-fra_Latn": {
          "accuracy": 0.5677777777777778,
          "correct": 511,
          "examples": 900,
          "multiclass_brier_score": 0.5662651315522427,
          "negative_log_likelihood": 1.095858989096891,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-spa_Latn": {
          "accuracy": 0.5544444444444444,
          "correct": 499,
          "examples": 900,
          "multiclass_brier_score": 0.5764217997719041,
          "negative_log_likelihood": 1.1263644125551366,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-zho_Hans": {
          "accuracy": 0.5588888888888889,
          "correct": 503,
          "examples": 900,
          "multiclass_brier_score": 0.6127871248165403,
          "negative_log_likelihood": 1.3210594793733768,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "mmlu-pro-test": {
          "accuracy": 0.3047835607209028,
          "correct": 3619,
          "examples": 11874,
          "multiclass_brier_score": 0.8366734653526192,
          "negative_log_likelihood": 2.0843591716629457,
          "uniform_baseline": {
            "accuracy": 0.11099517820714597,
            "multiclass_brier_score": 0.8890048217926751,
            "negative_log_likelihood": 2.228983477718964
          }
        }
      },
      "overall": {
        "accuracy": 0.3711536309723621,
        "correct": 6634,
        "examples": 17874,
        "multiclass_brier_score": 0.7686284226267893,
        "negative_log_likelihood": 1.8152825851701482,
        "uniform_baseline": {
          "accuracy": 0.15559203223268275,
          "multiclass_brier_score": 0.844407967767173,
          "negative_log_likelihood": 1.9599560598945625
        }
      }
    },
    "current": {
      "by_source_slice": {
        "bbh-logical-deduction-five-objects": {
          "accuracy": 0.512,
          "correct": 128,
          "examples": 250,
          "multiclass_brier_score": 0.6062119012997026,
          "negative_log_likelihood": 1.2335396517456851,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-logical-deduction-seven-objects": {
          "accuracy": 0.476,
          "correct": 119,
          "examples": 250,
          "multiclass_brier_score": 0.7233129909985951,
          "negative_log_likelihood": 1.610923808895743,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-logical-deduction-three-objects": {
          "accuracy": 0.768,
          "correct": 192,
          "examples": 250,
          "multiclass_brier_score": 0.39443245536276195,
          "negative_log_likelihood": 0.7049153410172034,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "bbh-tracking-shuffled-objects-five-objects": {
          "accuracy": 0.12,
          "correct": 30,
          "examples": 250,
          "multiclass_brier_score": 0.8831258364684099,
          "negative_log_likelihood": 1.7887631794247503,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-tracking-shuffled-objects-seven-objects": {
          "accuracy": 0.124,
          "correct": 31,
          "examples": 250,
          "multiclass_brier_score": 0.8979567273940374,
          "negative_log_likelihood": 2.053643360168717,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-tracking-shuffled-objects-three-objects": {
          "accuracy": 0.312,
          "correct": 78,
          "examples": 250,
          "multiclass_brier_score": 0.7349873447068362,
          "negative_log_likelihood": 1.23785525310506,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "belebele-deu_Latn": {
          "accuracy": 0.84,
          "correct": 756,
          "examples": 900,
          "multiclass_brier_score": 0.2231897826565889,
          "negative_log_likelihood": 0.4367418262828754,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-eng_Latn": {
          "accuracy": 0.8855555555555555,
          "correct": 797,
          "examples": 900,
          "multiclass_brier_score": 0.17000244944452955,
          "negative_log_likelihood": 0.36572167351472123,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-fra_Latn": {
          "accuracy": 0.8577777777777778,
          "correct": 772,
          "examples": 900,
          "multiclass_brier_score": 0.21197650360675677,
          "negative_log_likelihood": 0.44060015196594693,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-spa_Latn": {
          "accuracy": 0.8366666666666667,
          "correct": 753,
          "examples": 900,
          "multiclass_brier_score": 0.24268443558240538,
          "negative_log_likelihood": 0.480401114213536,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-zho_Hans": {
          "accuracy": 0.85,
          "correct": 765,
          "examples": 900,
          "multiclass_brier_score": 0.22587758493841034,
          "negative_log_likelihood": 0.45499554057264907,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "mmlu-pro-test": {
          "accuracy": 0.38630621526023246,
          "correct": 4587,
          "examples": 11874,
          "multiclass_brier_score": 0.7584534149447741,
          "negative_log_likelihood": 1.8928913921640595,
          "uniform_baseline": {
            "accuracy": 0.11099517820714597,
            "multiclass_brier_score": 0.8890048217926751,
            "negative_log_likelihood": 2.228983477718964
          }
        }
      },
      "overall": {
        "accuracy": 0.5039722501958152,
        "correct": 9008,
        "examples": 17874,
        "multiclass_brier_score": 0.6172228009241154,
        "negative_log_likelihood": 1.4878715908604745,
        "uniform_baseline": {
          "accuracy": 0.15559203223268275,
          "multiclass_brier_score": 0.844407967767173,
          "negative_log_likelihood": 1.9599560598945625
        }
      }
    },
    "full": {
      "by_source_slice": {
        "bbh-logical-deduction-five-objects": {
          "accuracy": 0.368,
          "correct": 92,
          "examples": 250,
          "multiclass_brier_score": 0.7328329117756509,
          "negative_log_likelihood": 1.4383312106897606,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-logical-deduction-seven-objects": {
          "accuracy": 0.216,
          "correct": 54,
          "examples": 250,
          "multiclass_brier_score": 0.8489287101207941,
          "negative_log_likelihood": 1.9046240975048534,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-logical-deduction-three-objects": {
          "accuracy": 0.784,
          "correct": 196,
          "examples": 250,
          "multiclass_brier_score": 0.32331820759509516,
          "negative_log_likelihood": 0.5707182013917134,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "bbh-tracking-shuffled-objects-five-objects": {
          "accuracy": 0.168,
          "correct": 42,
          "examples": 250,
          "multiclass_brier_score": 0.9422911023310142,
          "negative_log_likelihood": 1.9362256163457487,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-tracking-shuffled-objects-seven-objects": {
          "accuracy": 0.112,
          "correct": 28,
          "examples": 250,
          "multiclass_brier_score": 0.9723104177900325,
          "negative_log_likelihood": 2.2362308456866664,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-tracking-shuffled-objects-three-objects": {
          "accuracy": 0.312,
          "correct": 78,
          "examples": 250,
          "multiclass_brier_score": 0.7958847397765093,
          "negative_log_likelihood": 1.3158968952490755,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "belebele-deu_Latn": {
          "accuracy": 0.6788888888888889,
          "correct": 611,
          "examples": 900,
          "multiclass_brier_score": 0.44459978455089605,
          "negative_log_likelihood": 0.8522328572330174,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-eng_Latn": {
          "accuracy": 0.8211111111111111,
          "correct": 739,
          "examples": 900,
          "multiclass_brier_score": 0.26875799688689495,
          "negative_log_likelihood": 0.5444667601165114,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-fra_Latn": {
          "accuracy": 0.7266666666666667,
          "correct": 654,
          "examples": 900,
          "multiclass_brier_score": 0.3933355161167196,
          "negative_log_likelihood": 0.7790757924403804,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-spa_Latn": {
          "accuracy": 0.7044444444444444,
          "correct": 634,
          "examples": 900,
          "multiclass_brier_score": 0.4183569090729053,
          "negative_log_likelihood": 0.8145447577372547,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-zho_Hans": {
          "accuracy": 0.7633333333333333,
          "correct": 687,
          "examples": 900,
          "multiclass_brier_score": 0.3377493856469797,
          "negative_log_likelihood": 0.6590094251529196,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "mmlu-pro-test": {
          "accuracy": 0.28372915613946437,
          "correct": 3369,
          "examples": 11874,
          "multiclass_brier_score": 0.8522258404145246,
          "negative_log_likelihood": 2.1630162804256265,
          "uniform_baseline": {
            "accuracy": 0.11099517820714597,
            "multiclass_brier_score": 0.8890048217926751,
            "negative_log_likelihood": 2.228983477718964
          }
        }
      },
      "overall": {
        "accuracy": 0.40192458319346536,
        "correct": 7184,
        "examples": 17874,
        "multiclass_brier_score": 0.7245015544632623,
        "negative_log_likelihood": 1.7521852223286847,
        "uniform_baseline": {
          "accuracy": 0.15559203223268275,
          "multiclass_brier_score": 0.844407967767173,
          "negative_log_likelihood": 1.9599560598945625
        }
      }
    },
    "head_only": {
      "by_source_slice": {
        "bbh-logical-deduction-five-objects": {
          "accuracy": 0.484,
          "correct": 121,
          "examples": 250,
          "multiclass_brier_score": 0.6710955581455389,
          "negative_log_likelihood": 1.4749716877578096,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-logical-deduction-seven-objects": {
          "accuracy": 0.372,
          "correct": 93,
          "examples": 250,
          "multiclass_brier_score": 0.7676017825446775,
          "negative_log_likelihood": 2.4133005217822263,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-logical-deduction-three-objects": {
          "accuracy": 0.736,
          "correct": 184,
          "examples": 250,
          "multiclass_brier_score": 0.36304836661216383,
          "negative_log_likelihood": 0.6327291740662291,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "bbh-tracking-shuffled-objects-five-objects": {
          "accuracy": 0.144,
          "correct": 36,
          "examples": 250,
          "multiclass_brier_score": 0.9835637809217763,
          "negative_log_likelihood": 2.1598918016458937,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-tracking-shuffled-objects-seven-objects": {
          "accuracy": 0.136,
          "correct": 34,
          "examples": 250,
          "multiclass_brier_score": 1.012002245187751,
          "negative_log_likelihood": 3.5260994448009386,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-tracking-shuffled-objects-three-objects": {
          "accuracy": 0.324,
          "correct": 81,
          "examples": 250,
          "multiclass_brier_score": 0.7318082211159669,
          "negative_log_likelihood": 1.200557279697272,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "belebele-deu_Latn": {
          "accuracy": 0.8433333333333334,
          "correct": 759,
          "examples": 900,
          "multiclass_brier_score": 0.23445268950391898,
          "negative_log_likelihood": 0.4791979712688732,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-eng_Latn": {
          "accuracy": 0.8777777777777778,
          "correct": 790,
          "examples": 900,
          "multiclass_brier_score": 0.1891558662193704,
          "negative_log_likelihood": 0.41717113105686465,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-fra_Latn": {
          "accuracy": 0.8633333333333333,
          "correct": 777,
          "examples": 900,
          "multiclass_brier_score": 0.21514814687805345,
          "negative_log_likelihood": 0.46080486852824065,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-spa_Latn": {
          "accuracy": 0.8355555555555556,
          "correct": 752,
          "examples": 900,
          "multiclass_brier_score": 0.24834934531202657,
          "negative_log_likelihood": 0.53917748366581,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-zho_Hans": {
          "accuracy": 0.8444444444444444,
          "correct": 760,
          "examples": 900,
          "multiclass_brier_score": 0.2498921157734435,
          "negative_log_likelihood": 0.5310347444093398,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "mmlu-pro-test": {
          "accuracy": 0.32061647296614454,
          "correct": 3807,
          "examples": 11874,
          "multiclass_brier_score": 0.9127018380424604,
          "negative_log_likelihood": 3.122710538850784,
          "uniform_baseline": {
            "accuracy": 0.11099517820714597,
            "multiclass_brier_score": 0.8890048217926751,
            "negative_log_likelihood": 2.228983477718964
          }
        }
      },
      "overall": {
        "accuracy": 0.4584312409085823,
        "correct": 8194,
        "examples": 17874,
        "multiclass_brier_score": 0.7269217836447472,
        "negative_log_likelihood": 2.3562493003685914,
        "uniform_baseline": {
          "accuracy": 0.15559203223268275,
          "multiclass_brier_score": 0.844407967767173,
          "negative_log_likelihood": 1.9599560598945625
        }
      }
    },
    "original": {
      "by_source_slice": {
        "bbh-logical-deduction-five-objects": {
          "accuracy": 0.588,
          "correct": 147,
          "examples": 250,
          "multiclass_brier_score": 0.7444592224345026,
          "negative_log_likelihood": 3.400003504282403,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-logical-deduction-seven-objects": {
          "accuracy": 0.536,
          "correct": 134,
          "examples": 250,
          "multiclass_brier_score": 0.8365528979666588,
          "negative_log_likelihood": 4.456906394594916,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-logical-deduction-three-objects": {
          "accuracy": 0.828,
          "correct": 207,
          "examples": 250,
          "multiclass_brier_score": 0.3330000339468915,
          "negative_log_likelihood": 1.8118430721865137,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "bbh-tracking-shuffled-objects-five-objects": {
          "accuracy": 0.156,
          "correct": 39,
          "examples": 250,
          "multiclass_brier_score": 1.4303703270240207,
          "negative_log_likelihood": 5.789846177966438,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-tracking-shuffled-objects-seven-objects": {
          "accuracy": 0.128,
          "correct": 32,
          "examples": 250,
          "multiclass_brier_score": 1.471290473852679,
          "negative_log_likelihood": 6.7765860941776985,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-tracking-shuffled-objects-three-objects": {
          "accuracy": 0.308,
          "correct": 77,
          "examples": 250,
          "multiclass_brier_score": 1.2417013668258128,
          "negative_log_likelihood": 4.6156168805968205,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "belebele-deu_Latn": {
          "accuracy": 0.8555555555555555,
          "correct": 770,
          "examples": 900,
          "multiclass_brier_score": 0.27731957109137206,
          "negative_log_likelihood": 2.069691894599715,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-eng_Latn": {
          "accuracy": 0.8866666666666667,
          "correct": 798,
          "examples": 900,
          "multiclass_brier_score": 0.21783017915243724,
          "negative_log_likelihood": 1.7637330682627794,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-fra_Latn": {
          "accuracy": 0.8533333333333334,
          "correct": 768,
          "examples": 900,
          "multiclass_brier_score": 0.2847288696874747,
          "negative_log_likelihood": 2.19432933407547,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-spa_Latn": {
          "accuracy": 0.8388888888888889,
          "correct": 755,
          "examples": 900,
          "multiclass_brier_score": 0.3082504635731747,
          "negative_log_likelihood": 2.3875321711943647,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-zho_Hans": {
          "accuracy": 0.86,
          "correct": 774,
          "examples": 900,
          "multiclass_brier_score": 0.2695231920442741,
          "negative_log_likelihood": 2.132001907153469,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "mmlu-pro-test": {
          "accuracy": 0.3995283813373758,
          "correct": 4744,
          "examples": 11874,
          "multiclass_brier_score": 1.1006630796645316,
          "negative_log_likelihood": 7.711696327114687,
          "uniform_baseline": {
            "accuracy": 0.11099517820714597,
            "multiclass_brier_score": 0.8890048217926751,
            "negative_log_likelihood": 2.228983477718964
          }
        }
      },
      "overall": {
        "accuracy": 0.5172317332438179,
        "correct": 9245,
        "examples": 17874,
        "multiclass_brier_score": 0.8842734718833564,
        "negative_log_likelihood": 6.029648777938189,
        "uniform_baseline": {
          "accuracy": 0.15559203223268275,
          "multiclass_brier_score": 0.844407967767173,
          "negative_log_likelihood": 1.9599560598945625
        }
      }
    },
    "previous": {
      "by_source_slice": {
        "bbh-logical-deduction-five-objects": {
          "accuracy": 0.44,
          "correct": 110,
          "examples": 250,
          "multiclass_brier_score": 0.6559344238264133,
          "negative_log_likelihood": 1.3693863224401994,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-logical-deduction-seven-objects": {
          "accuracy": 0.424,
          "correct": 106,
          "examples": 250,
          "multiclass_brier_score": 0.7240901923520918,
          "negative_log_likelihood": 1.6284158271546922,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-logical-deduction-three-objects": {
          "accuracy": 0.724,
          "correct": 181,
          "examples": 250,
          "multiclass_brier_score": 0.4261787195983975,
          "negative_log_likelihood": 0.7563939898289093,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "bbh-tracking-shuffled-objects-five-objects": {
          "accuracy": 0.176,
          "correct": 44,
          "examples": 250,
          "multiclass_brier_score": 0.8804946997941235,
          "negative_log_likelihood": 1.806204397481368,
          "uniform_baseline": {
            "accuracy": 0.20000000000000068,
            "multiclass_brier_score": 0.8000000000000027,
            "negative_log_likelihood": 1.6094379124340932
          }
        },
        "bbh-tracking-shuffled-objects-seven-objects": {
          "accuracy": 0.152,
          "correct": 38,
          "examples": 250,
          "multiclass_brier_score": 0.8987279122308649,
          "negative_log_likelihood": 2.070601515797062,
          "uniform_baseline": {
            "accuracy": 0.14285714285714285,
            "multiclass_brier_score": 0.8571428571428596,
            "negative_log_likelihood": 1.945910149055306
          }
        },
        "bbh-tracking-shuffled-objects-three-objects": {
          "accuracy": 0.312,
          "correct": 78,
          "examples": 250,
          "multiclass_brier_score": 0.7395744607622045,
          "negative_log_likelihood": 1.2165784241698112,
          "uniform_baseline": {
            "accuracy": 0.3333333333333329,
            "multiclass_brier_score": 0.6666666666666659,
            "negative_log_likelihood": 1.0986122886681078
          }
        },
        "belebele-deu_Latn": {
          "accuracy": 0.8322222222222222,
          "correct": 749,
          "examples": 900,
          "multiclass_brier_score": 0.24979657253623078,
          "negative_log_likelihood": 0.5026164867176588,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-eng_Latn": {
          "accuracy": 0.8766666666666667,
          "correct": 789,
          "examples": 900,
          "multiclass_brier_score": 0.18377990158850463,
          "negative_log_likelihood": 0.39123789337091497,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-fra_Latn": {
          "accuracy": 0.8455555555555555,
          "correct": 761,
          "examples": 900,
          "multiclass_brier_score": 0.22466499788935326,
          "negative_log_likelihood": 0.4665799834560347,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-spa_Latn": {
          "accuracy": 0.8177777777777778,
          "correct": 736,
          "examples": 900,
          "multiclass_brier_score": 0.2648695207858629,
          "negative_log_likelihood": 0.5372960919154716,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "belebele-zho_Hans": {
          "accuracy": 0.8388888888888889,
          "correct": 755,
          "examples": 900,
          "multiclass_brier_score": 0.24032897543772283,
          "negative_log_likelihood": 0.48899801897125417,
          "uniform_baseline": {
            "accuracy": 0.25,
            "multiclass_brier_score": 0.75,
            "negative_log_likelihood": 1.3862943611198737
          }
        },
        "mmlu-pro-test": {
          "accuracy": 0.3735893548930436,
          "correct": 4436,
          "examples": 11874,
          "multiclass_brier_score": 0.7670399836453746,
          "negative_log_likelihood": 1.939659610521011,
          "uniform_baseline": {
            "accuracy": 0.11099517820714597,
            "multiclass_brier_score": 0.8890048217926751,
            "negative_log_likelihood": 2.228983477718964
          }
        }
      },
      "overall": {
        "accuracy": 0.49138413337809106,
        "correct": 8783,
        "examples": 17874,
        "multiclass_brier_score": 0.6286325858431332,
        "negative_log_likelihood": 1.5324756048748365,
        "uniform_baseline": {
          "accuracy": 0.15559203223268275,
          "multiclass_brier_score": 0.844407967767173,
          "negative_log_likelihood": 1.9599560598945625
        }
      }
    }
  },
  "authoring_content_audit": {
    "external_benchmark_item_files_read": false,
    "handling": "not opened, reproduced, or used; recorded as accidental exposure",
    "unsolicited_web_search_diagnostic_snippet": true
  },
  "calibration_questions": 64,
  "claim_scope": "descriptive exploratory reused external suite; internal test deferred",
  "completed_at": "2026-10-09T12:40:46.729400+00:00",
  "development_questions": 128,
  "development_soft_cross_entropy": {
    "0": 1.3386019859772205,
    "1080": 1.1418300499457246,
    "1438": 1.1440039756173306,
    "360": 1.1730923340408026,
    "720": 1.1583489514823242
  },
  "external_probabilities": "uncalibrated masked softmax",
  "report_date": "2026-10-09",
  "schema_version": "horscht-paper-evidence-v1",
  "selected_update": 1080,
  "sources": {
    "attention": {
      "path": "data/external_evaluation/results/current-corpus-option-attention-2026-10-06/blind-aggregate-result.json",
      "sha256": "f5ef94e240b7dbaf8547232faa85bff86a45f87e37623c4aeebeb0dcdc13ec2f"
    },
    "calibration": {
      "path": "runs/new-data-last-block-2026-10-08/fast-evaluation-003/calibration.json",
      "sha256": "02c1fa9a38eca35636b21d89926e618d8ee514b08052ba353be8a5f4966b74e9"
    },
    "checkpoint": {
      "path": "runs/new-data-last-block-2026-10-08/fast-evaluation-003/selected-checkpoint.json",
      "sha256": "34bd4887c03fb95642b75badac53fb7254cb7a22ea4fad826c898dd3881b38f9"
    },
    "completion": {
      "path": "runs/new-data-last-block-2026-10-08/fast-evaluation-003/benchmark-report.json",
      "sha256": "269fb2184a9291c18949735507a7ed2abc7d6b3887a5a03d697042214e59820c"
    },
    "current": {
      "path": "data/external_evaluation/results/new-data-last-block-fast-2026-10-09/blind-aggregate-result.json",
      "sha256": "dcaceb9121c5d81b8a1b3d4c66881e7b3a5ef5a2d671559dfad3d297350a5e93"
    },
    "full": {
      "path": "data/external_evaluation/results/current-corpus-benchmark-2026-10-04-final/blind-aggregate-result.json",
      "sha256": "e5504880e813360af002511796b3500839107f94cfb4bc87a03682bfa9239119"
    },
    "head_only": {
      "path": "data/external_evaluation/results/current-corpus-head-only-2026-10-04/blind-aggregate-result.json",
      "sha256": "5f9184205d466a728fc7b5bffd3862bfa7485e2676ebd50cb04414ccac336559"
    },
    "original": {
      "path": "data/external_evaluation/results/original-qwen-label-likelihood-2026-10-04/blind-aggregate-result.json",
      "sha256": "5b31638b803f6deeaf8fca368a76d8191f22b4dbe90dfd11b723072b841e0046"
    },
    "previous": {
      "path": "data/external_evaluation/results/current-corpus-last-block-2026-10-05/blind-aggregate-result.json",
      "sha256": "159c1e1ce9929ad6d664d7b614c3ace378da2c89e2bfc953ef76755980e6add7"
    },
    "protocol": {
      "path": "runs/new-data-last-block-2026-10-08/fast-evaluation-003/blind-protocol.json",
      "sha256": "2b7c8fe68d2345aed5f6125a267868cad226d001bfd35984b6b4a776e9b1eef7"
    },
    "selection": {
      "path": "runs/new-data-last-block-2026-10-08/fast-evaluation-003/development-selection.json",
      "sha256": "8f5de4bdc02640c3dcb571162cfbb3379f02d2d6d9be90f324f7bc5cc101741e"
    },
    "snapshot_manifest": {
      "path": "data/training_snapshots/2026-10-08-v4/manifest.json",
      "sha256": "6413df891b6ee10073c8e49f4c3c0b19375c495ff595ab7889a4386e7bb427fd"
    },
    "training_manifest": {
      "path": "runs/new-data-last-block-2026-10-08/main-recovery-002/manifest.json",
      "sha256": "ae8cf90c1af5d8724ea7dd2341d445e079083196efd2a142c8cf306a9d1e2b36"
    }
  },
  "temperature": 0.9174703667456412
}