[
 {
  "base_model": "gpt-4o-2024-05-13",
  "candidate_model": "gpt-4o-2024-08-06",
  "questions": 4551,
  "base": 0.7732366512854317,
  "candidate": 0.7747747747747747,
  "broke": 157,
  "fixed": 164,
  "output_tokens": {
   "base": 70.23159745110965,
   "candidate": 57.065260382333555
  },
  "tasks": [
   {
    "task": "gsm8k",
    "questions": 1000,
    "base": 0.905,
    "candidate": 0.909,
    "broke": 20,
    "fixed": 24,
    "p_adjusted": 1.0,
    "verdict": "same"
   },
   {
    "task": "legalbench",
    "questions": 2047,
    "base": 0.6472887151929653,
    "candidate": 0.6477772349780166,
    "broke": 98,
    "fixed": 99,
    "p_adjusted": 1.0,
    "verdict": "same"
   },
   {
    "task": "math",
    "questions": 437,
    "base": 0.8649885583524027,
    "candidate": 0.8787185354691075,
    "broke": 13,
    "fixed": 19,
    "p_adjusted": 1.0,
    "verdict": "same"
   },
   {
    "task": "mmlu",
    "questions": 567,
    "base": 0.7548500881834215,
    "candidate": 0.746031746031746,
    "broke": 23,
    "fixed": 18,
    "p_adjusted": 1.0,
    "verdict": "same"
   },
   {
    "task": "openbookqa",
    "questions": 500,
    "base": 0.966,
    "candidate": 0.968,
    "broke": 3,
    "fixed": 4,
    "p_adjusted": 1.0,
    "verdict": "same"
   }
  ],
  "regressed": [],
  "improved": []
 },
 {
  "base_model": "gemini-1.5-pro-001",
  "candidate_model": "gemini-1.5-pro-002",
  "questions": 4551,
  "base": 0.7778510217534608,
  "candidate": 0.7864205669083718,
  "broke": 263,
  "fixed": 302,
  "output_tokens": {
   "base": 0.0,
   "candidate": 0.0
  },
  "tasks": [
   {
    "task": "gsm8k",
    "questions": 1000,
    "base": 0.836,
    "candidate": 0.817,
    "broke": 112,
    "fixed": 93,
    "p_adjusted": 0.5775793732843882,
    "verdict": "same"
   },
   {
    "task": "legalbench",
    "questions": 2047,
    "base": 0.701514411333659,
    "candidate": 0.6971177332681974,
    "broke": 115,
    "fixed": 106,
    "p_adjusted": 0.59058206149012,
    "verdict": "same"
   },
   {
    "task": "math",
    "questions": 437,
    "base": 0.8627002288329519,
    "candidate": 0.9336384439359268,
    "broke": 5,
    "fixed": 36,
    "p_adjusted": 3.920786184608005e-06,
    "verdict": "improved"
   },
   {
    "task": "mmlu",
    "questions": 567,
    "base": 0.7760141093474426,
    "candidate": 0.7954144620811288,
    "broke": 24,
    "fixed": 35,
    "p_adjusted": 0.5775793732843882,
    "verdict": "same"
   },
   {
    "task": "openbookqa",
    "questions": 500,
    "base": 0.902,
    "candidate": 0.952,
    "broke": 7,
    "fixed": 32,
    "p_adjusted": 0.0002810191363096237,
    "verdict": "improved"
   }
  ],
  "regressed": [],
  "improved": [
   "math",
   "openbookqa"
  ]
 },
 {
  "base_model": "gemini-1.5-flash-001",
  "candidate_model": "gemini-1.5-flash-002",
  "questions": 4551,
  "base": 0.7084157328059767,
  "candidate": 0.6262359920896506,
  "broke": 688,
  "fixed": 314,
  "output_tokens": {
   "base": 0.0,
   "candidate": 0.0
  },
  "tasks": [
   {
    "task": "gsm8k",
    "questions": 1000,
    "base": 0.785,
    "candidate": 0.328,
    "broke": 488,
    "fixed": 31,
    "p_adjusted": 4.487578861421274e-106,
    "verdict": "regressed"
   },
   {
    "task": "legalbench",
    "questions": 2047,
    "base": 0.6023448949682462,
    "candidate": 0.6223742061553493,
    "broke": 114,
    "fixed": 155,
    "p_adjusted": 0.04374453544981259,
    "verdict": "improved"
   },
   {
    "task": "math",
    "questions": 437,
    "base": 0.7871853546910755,
    "candidate": 0.9290617848970252,
    "broke": 7,
    "fixed": 69,
    "p_adjusted": 2.5671868022053817e-13,
    "verdict": "improved"
   },
   {
    "task": "mmlu",
    "questions": 567,
    "base": 0.7019400352733686,
    "candidate": 0.6790123456790124,
    "broke": 60,
    "fixed": 47,
    "p_adjusted": 0.4917826292253687,
    "verdict": "same"
   },
   {
    "task": "openbookqa",
    "questions": 500,
    "base": 0.928,
    "candidate": 0.914,
    "broke": 19,
    "fixed": 12,
    "p_adjusted": 0.4917826292253687,
    "verdict": "same"
   }
  ],
  "regressed": [
   "gsm8k"
  ],
  "improved": [
   "legalbench",
   "math"
  ]
 },
 {
  "base_model": "gemini-1.0-pro-001",
  "candidate_model": "gemini-1.0-pro-002",
  "questions": 4551,
  "base": 0.6084377059986816,
  "candidate": 0.5750384530872336,
  "broke": 700,
  "fixed": 548,
  "output_tokens": {
   "base": 0.0,
   "candidate": 0.0
  },
  "tasks": [
   {
    "task": "gsm8k",
    "questions": 1000,
    "base": 0.783,
    "candidate": 0.816,
    "broke": 81,
    "fixed": 114,
    "p_adjusted": 0.03149858229825494,
    "verdict": "improved"
   },
   {
    "task": "legalbench",
    "questions": 2047,
    "base": 0.4425989252564729,
    "candidate": 0.3873961895456766,
    "broke": 411,
    "fixed": 298,
    "p_adjusted": 0.00010009907643725127,
    "verdict": "regressed"
   },
   {
    "task": "math",
    "questions": 437,
    "base": 0.6567505720823799,
    "candidate": 0.7162471395881007,
    "broke": 41,
    "fixed": 67,
    "p_adjusted": 0.03149858229825494,
    "verdict": "improved"
   },
   {
    "task": "mmlu",
    "questions": 567,
    "base": 0.6190476190476191,
    "candidate": 0.5308641975308642,
    "broke": 97,
    "fixed": 47,
    "p_adjusted": 0.00011267129112625446,
    "verdict": "regressed"
   },
   {
    "task": "openbookqa",
    "questions": 500,
    "base": 0.884,
    "candidate": 0.788,
    "broke": 70,
    "fixed": 22,
    "p_adjusted": 2.6740715855209355e-06,
    "verdict": "regressed"
   }
  ],
  "regressed": [
   "legalbench",
   "mmlu",
   "openbookqa"
  ],
  "improved": [
   "gsm8k",
   "math"
  ]
 },
 {
  "base_model": "mistral-large-2402",
  "candidate_model": "mistral-large-2407",
  "questions": 4066,
  "base": 0.6276438760452533,
  "candidate": 0.6871618298081653,
  "broke": 331,
  "fixed": 573,
  "output_tokens": {
   "base": 46.1588785046729,
   "candidate": 71.90457452041318
  },
  "tasks": [
   {
    "task": "gsm8k",
    "questions": 1000,
    "base": 0.694,
    "candidate": 0.912,
    "broke": 31,
    "fixed": 249,
    "p_adjusted": 9.262127869233525e-43,
    "verdict": "improved"
   },
   {
    "task": "legalbench",
    "questions": 1562,
    "base": 0.44814340588988477,
    "candidate": 0.4379001280409731,
    "broke": 184,
    "fixed": 168,
    "p_adjusted": 0.4240390178062495,
    "verdict": "same"
   },
   {
    "task": "math",
    "questions": 437,
    "base": 0.7940503432494279,
    "candidate": 0.7208237986270023,
    "broke": 78,
    "fixed": 46,
    "p_adjusted": 0.015483674705964155,
    "verdict": "regressed"
   },
   {
    "task": "mmlu",
    "questions": 567,
    "base": 0.6419753086419753,
    "candidate": 0.7354497354497355,
    "broke": 26,
    "fixed": 79,
    "p_adjusted": 8.663410468053857e-07,
    "verdict": "improved"
   },
   {
    "task": "openbookqa",
    "questions": 500,
    "base": 0.894,
    "candidate": 0.932,
    "broke": 12,
    "fixed": 31,
    "p_adjusted": 0.015483674705964155,
    "verdict": "improved"
   }
  ],
  "regressed": [
   "math"
  ],
  "improved": [
   "gsm8k",
   "mmlu",
   "openbookqa"
  ]
 },
 {
  "base_model": "llama-3-70b",
  "candidate_model": "llama-3.1-70b-instruct",
  "questions": 4551,
  "base": 0.7470885519666007,
  "candidate": 0.7400571303010327,
  "broke": 560,
  "fixed": 528,
  "output_tokens": {
   "base": 1.0,
   "candidate": 51.128103713469564
  },
  "tasks": [
   {
    "task": "gsm8k",
    "questions": 1000,
    "base": 0.805,
    "candidate": 0.938,
    "broke": 18,
    "fixed": 151,
    "p_adjusted": 1.170354170389962e-26,
    "verdict": "improved"
   },
   {
    "task": "legalbench",
    "questions": 2047,
    "base": 0.6917440156326331,
    "candidate": 0.5828041035661944,
    "broke": 463,
    "fixed": 240,
    "p_adjusted": 1.225873533466912e-16,
    "verdict": "regressed"
   },
   {
    "task": "math",
    "questions": 437,
    "base": 0.7139588100686499,
    "candidate": 0.8329519450800915,
    "broke": 31,
    "fixed": 83,
    "p_adjusted": 3.5410138328749065e-06,
    "verdict": "improved"
   },
   {
    "task": "mmlu",
    "questions": 567,
    "base": 0.7054673721340388,
    "candidate": 0.7125220458553791,
    "broke": 33,
    "fixed": 37,
    "p_adjusted": 1.0,
    "verdict": "same"
   },
   {
    "task": "openbookqa",
    "questions": 500,
    "base": 0.934,
    "candidate": 0.938,
    "broke": 15,
    "fixed": 17,
    "p_adjusted": 1.0,
    "verdict": "same"
   }
  ],
  "regressed": [
   "legalbench"
  ],
  "improved": [
   "gsm8k",
   "math"
  ]
 }
]