[
 {
  "answerer": "deepseek-v3",
  "panel": [
   "gpt-4o",
   "llama-3.1-70b",
   "gpt-4o-mini"
  ],
  "accuracy": 0.7868600307624698,
  "wrong": 970,
  "auroc": 0.7915332064705763,
  "ece": 0.05422822652287598,
  "arbiter": {
   "flagged": 0.12392880685563612,
   "precision": 0.5585106382978723,
   "recall": 0.3247422680412371
  },
  "majority": {
   "flagged": 0.15117556580971214,
   "precision": 0.4331395348837209,
   "recall": 0.30721649484536084
  },
  "best_critic": {
   "flagged": 0.14238628872775214,
   "precision": 0.5154320987654321,
   "recall": 0.3443298969072165
  },
  "split_panel": 0.22654361678751922,
  "accuracy_when_split": 0.6401551891367604,
  "accuracy_when_all_pass": 0.8594327990135635,
  "bins": {
   "right": [
    3,
    19,
    16,
    1,
    0,
    15,
    5,
    7,
    4,
    2,
    3,
    47,
    34,
    11,
    4,
    26,
    17,
    21,
    7,
    7,
    12,
    2,
    81,
    36,
    70,
    18,
    66,
    434,
    375,
    171,
    20,
    7,
    13,
    12,
    10,
    12,
    7,
    268,
    127,
    1591
   ],
   "wrong": [
    13,
    22,
    17,
    0,
    1,
    11,
    7,
    9,
    0,
    5,
    0,
    46,
    28,
    25,
    4,
    42,
    16,
    47,
    17,
    5,
    3,
    5,
    34,
    12,
    57,
    11,
    20,
    141,
    196,
    104,
    8,
    0,
    2,
    3,
    3,
    1,
    4,
    19,
    14,
    18
   ]
  },
  "patterns": [
   {
    "verdicts": "vvv",
    "n": 3244,
    "right": 2788,
    "p": 0.8733
   },
   {
    "verdicts": "vvx",
    "n": 220,
    "right": 189,
    "p": 0.8088
   },
   {
    "verdicts": "vxv",
    "n": 232,
    "right": 138,
    "p": 0.6872
   },
   {
    "verdicts": "vxx",
    "n": 207,
    "right": 152,
    "p": 0.5527
   },
   {
    "verdicts": "xvv",
    "n": 167,
    "right": 76,
    "p": 0.5674
   },
   {
    "verdicts": "xvx",
    "n": 139,
    "right": 80,
    "p": 0.4781
   },
   {
    "verdicts": "xxv",
    "n": 66,
    "right": 25,
    "p": 0.3329
   },
   {
    "verdicts": "xxx",
    "n": 276,
    "right": 133,
    "p": 0.1958
   }
  ]
 },
 {
  "answerer": "gemini-1.5-flash",
  "panel": [
   "deepseek-v3",
   "gpt-4o",
   "llama-3.1-70b"
  ],
  "accuracy": 0.6262359920896506,
  "wrong": 1701,
  "auroc": 0.8903811999133636,
  "ece": 0.055445579145401436,
  "arbiter": {
   "flagged": 0.3137771918259723,
   "precision": 0.8011204481792717,
   "recall": 0.6725455614344503
  },
  "majority": {
   "flagged": 0.3082838936497473,
   "precision": 0.7997148966500356,
   "recall": 0.6596119929453262
  },
  "best_critic": {
   "flagged": 0.3069655020874533,
   "precision": 0.8017179670722978,
   "recall": 0.6584362139917695
  },
  "split_panel": 0.1916062403867282,
  "accuracy_when_split": 0.5573394495412844,
  "accuracy_when_all_pass": 0.8567619047619047,
  "bins": {
   "right": [
    17,
    15,
    10,
    1,
    0,
    25,
    61,
    10,
    0,
    8,
    5,
    10,
    9,
    0,
    51,
    0,
    45,
    1,
    0,
    16,
    15,
    17,
    26,
    0,
    0,
    78,
    52,
    5,
    14,
    11,
    379,
    512,
    2,
    5,
    3,
    5,
    0,
    0,
    82,
    1360
   ],
   "wrong": [
    780,
    29,
    21,
    9,
    0,
    27,
    106,
    26,
    0,
    6,
    8,
    36,
    6,
    0,
    33,
    0,
    35,
    0,
    2,
    20,
    33,
    21,
    29,
    0,
    0,
    44,
    30,
    2,
    9,
    1,
    106,
    244,
    0,
    0,
    1,
    0,
    1,
    0,
    3,
    33
   ]
  },
  "patterns": [
   {
    "verdicts": "vvv",
    "n": 2625,
    "right": 2249,
    "p": 0.8933
   },
   {
    "verdicts": "vvx",
    "n": 273,
    "right": 189,
    "p": 0.7138
   },
   {
    "verdicts": "vxv",
    "n": 147,
    "right": 75,
    "p": 0.6139
   },
   {
    "verdicts": "vxx",
    "n": 109,
    "right": 60,
    "p": 0.3215
   },
   {
    "verdicts": "xvv",
    "n": 103,
    "right": 56,
    "p": 0.6197
   },
   {
    "verdicts": "xvx",
    "n": 129,
    "right": 72,
    "p": 0.3434
   },
   {
    "verdicts": "xxv",
    "n": 111,
    "right": 34,
    "p": 0.2263
   },
   {
    "verdicts": "xxx",
    "n": 1054,
    "right": 115,
    "p": 0.0384
   }
  ]
 },
 {
  "answerer": "gpt-4o",
  "panel": [
   "deepseek-v3",
   "llama-3.1-70b",
   "gpt-4o-mini"
  ],
  "accuracy": 0.7747747747747747,
  "wrong": 1025,
  "auroc": 0.8098644217865888,
  "ece": 0.043979753050671054,
  "arbiter": {
   "flagged": 0.11667765326301911,
   "precision": 0.6026365348399246,
   "recall": 0.3121951219512195
  },
  "majority": {
   "flagged": 0.15007690617446715,
   "precision": 0.5124450951683748,
   "recall": 0.34146341463414637
  },
  "best_critic": {
   "flagged": 0.14238628872775214,
   "precision": 0.6003086419753086,
   "recall": 0.3795121951219512
  },
  "split_panel": 0.22588442100637224,
  "accuracy_when_split": 0.6215953307392996,
  "accuracy_when_all_pass": 0.8594327990135635,
  "bins": {
   "right": [
    14,
    16,
    0,
    2,
    1,
    0,
    3,
    3,
    8,
    6,
    2,
    0,
    4,
    33,
    5,
    36,
    40,
    14,
    19,
    5,
    55,
    3,
    42,
    46,
    97,
    10,
    9,
    482,
    192,
    338,
    3,
    6,
    15,
    18,
    5,
    1,
    10,
    261,
    100,
    1622
   ],
   "wrong": [
    71,
    66,
    1,
    2,
    17,
    2,
    3,
    3,
    4,
    7,
    1,
    1,
    1,
    21,
    2,
    32,
    48,
    10,
    14,
    14,
    73,
    1,
    17,
    19,
    71,
    4,
    7,
    160,
    92,
    194,
    0,
    2,
    3,
    5,
    1,
    1,
    1,
    19,
    16,
    19
   ]
  },
  "patterns": [
   {
    "verdicts": "vvv",
    "n": 3244,
    "right": 2788,
    "p": 0.8742
   },
   {
    "verdicts": "vvx",
    "n": 220,
    "right": 189,
    "p": 0.8212
   },
   {
    "verdicts": "vxv",
    "n": 232,
    "right": 138,
    "p": 0.6685
   },
   {
    "verdicts": "vxx",
    "n": 207,
    "right": 152,
    "p": 0.5446
   },
   {
    "verdicts": "xvv",
    "n": 172,
    "right": 78,
    "p": 0.5779
   },
   {
    "verdicts": "xvx",
    "n": 60,
    "right": 33,
    "p": 0.3595
   },
   {
    "verdicts": "xxv",
    "n": 137,
    "right": 49,
    "p": 0.2538
   },
   {
    "verdicts": "xxx",
    "n": 279,
    "right": 99,
    "p": 0.2073
   }
  ]
 },
 {
  "answerer": "gpt-4o-mini",
  "panel": [
   "deepseek-v3",
   "gpt-4o",
   "llama-3.1-70b"
  ],
  "accuracy": 0.7088551966600747,
  "wrong": 1325,
  "auroc": 0.8664416474634163,
  "ece": 0.049682558179863515,
  "arbiter": {
   "flagged": 0.21621621621621623,
   "precision": 0.7591463414634146,
   "recall": 0.5637735849056604
  },
  "majority": {
   "flagged": 0.16172269830806416,
   "precision": 0.7961956521739131,
   "recall": 0.4422641509433962
  },
  "best_critic": {
   "flagged": 0.18501428257525818,
   "precision": 0.7589073634204275,
   "recall": 0.48226415094339625
  },
  "split_panel": 0.2074269391342562,
  "accuracy_when_split": 0.4311440677966102,
  "accuracy_when_all_pass": 0.8581997533908755,
  "bins": {
   "right": [
    18,
    22,
    10,
    0,
    28,
    15,
    4,
    8,
    16,
    12,
    15,
    0,
    1,
    14,
    29,
    0,
    0,
    45,
    0,
    0,
    0,
    68,
    9,
    6,
    19,
    8,
    19,
    9,
    3,
    399,
    517,
    5,
    8,
    5,
    0,
    0,
    0,
    7,
    334,
    1573
   ],
   "wrong": [
    260,
    95,
    72,
    0,
    62,
    38,
    4,
    20,
    13,
    12,
    12,
    0,
    0,
    19,
    70,
    5,
    0,
    65,
    0,
    0,
    0,
    52,
    14,
    3,
    9,
    5,
    12,
    6,
    3,
    129,
    280,
    2,
    2,
    3,
    0,
    0,
    0,
    0,
    35,
    23
   ]
  },
  "patterns": [
   {
    "verdicts": "vvv",
    "n": 3244,
    "right": 2784,
    "p": 0.893
   },
   {
    "verdicts": "vvx",
    "n": 232,
    "right": 138,
    "p": 0.615
   },
   {
    "verdicts": "vxv",
    "n": 167,
    "right": 76,
    "p": 0.4862
   },
   {
    "verdicts": "vxx",
    "n": 66,
    "right": 25,
    "p": 0.1464
   },
   {
    "verdicts": "xvv",
    "n": 172,
    "right": 78,
    "p": 0.5027
   },
   {
    "verdicts": "xvx",
    "n": 137,
    "right": 49,
    "p": 0.141
   },
   {
    "verdicts": "xxv",
    "n": 170,
    "right": 41,
    "p": 0.1047
   },
   {
    "verdicts": "xxx",
    "n": 363,
    "right": 35,
    "p": 0.0211
   }
  ]
 },
 {
  "answerer": "llama-3.1-70b",
  "panel": [
   "deepseek-v3",
   "gpt-4o",
   "gpt-4o-mini"
  ],
  "accuracy": 0.7400571303010327,
  "wrong": 1183,
  "auroc": 0.848755780123403,
  "ece": 0.060409011364701476,
  "arbiter": {
   "flagged": 0.2188529993408042,
   "precision": 0.6736947791164659,
   "recall": 0.5672020287404903
  },
  "majority": {
   "flagged": 0.16435948143265217,
   "precision": 0.6818181818181818,
   "recall": 0.43110735418427726
  },
  "best_critic": {
   "flagged": 0.17161063502526916,
   "precision": 0.6978233034571063,
   "recall": 0.4606931530008453
  },
  "split_panel": 0.2039112283014722,
  "accuracy_when_split": 0.5355603448275862,
  "accuracy_when_all_pass": 0.8594327990135635,
  "bins": {
   "right": [
    35,
    1,
    28,
    23,
    6,
    36,
    2,
    7,
    6,
    17,
    19,
    12,
    4,
    0,
    0,
    27,
    23,
    45,
    34,
    0,
    0,
    0,
    0,
    25,
    58,
    8,
    2,
    3,
    2,
    203,
    375,
    347,
    14,
    7,
    0,
    2,
    18,
    157,
    220,
    1602
   ],
   "wrong": [
    161,
    14,
    79,
    75,
    8,
    95,
    5,
    3,
    11,
    20,
    18,
    10,
    8,
    1,
    0,
    33,
    61,
    40,
    29,
    0,
    0,
    0,
    0,
    9,
    20,
    5,
    2,
    0,
    0,
    60,
    154,
    198,
    4,
    1,
    0,
    0,
    1,
    9,
    28,
    21
   ]
  },
  "patterns": [
   {
    "verdicts": "vvv",
    "n": 3244,
    "right": 2788,
    "p": 0.8986
   },
   {
    "verdicts": "vvx",
    "n": 220,
    "right": 188,
    "p": 0.7745
   },
   {
    "verdicts": "vxv",
    "n": 167,
    "right": 76,
    "p": 0.527
   },
   {
    "verdicts": "vxx",
    "n": 139,
    "right": 80,
    "p": 0.2996
   },
   {
    "verdicts": "xvv",
    "n": 172,
    "right": 78,
    "p": 0.5276
   },
   {
    "verdicts": "xvx",
    "n": 60,
    "right": 34,
    "p": 0.2378
   },
   {
    "verdicts": "xxv",
    "n": 170,
    "right": 41,
    "p": 0.1263
   },
   {
    "verdicts": "xxx",
    "n": 379,
    "right": 83,
    "p": 0.0415
   }
  ]
 },
 {
  "answerer": "llama-3.1-8b",
  "panel": [
   "deepseek-v3",
   "gpt-4o",
   "llama-3.1-70b"
  ],
  "accuracy": 0.4919797846627115,
  "wrong": 2312,
  "auroc": 0.9575554691834436,
  "ece": 0.08281283035302597,
  "arbiter": {
   "flagged": 0.46231597451109646,
   "precision": 0.9296577946768061,
   "recall": 0.8460207612456747
  },
  "majority": {
   "flagged": 0.45726214018896943,
   "precision": 0.9336857280153772,
   "recall": 0.8403979238754326
  },
  "best_critic": {
   "flagged": 0.46275543836519445,
   "precision": 0.9320987654320988,
   "recall": 0.8490484429065744
  },
  "split_panel": 0.12326961107448912,
  "accuracy_when_split": 0.47058823529411764,
  "accuracy_when_all_pass": 0.8922934076137419,
  "bins": {
   "right": [
    55,
    18,
    28,
    23,
    8,
    2,
    1,
    0,
    0,
    0,
    0,
    1,
    1,
    1,
    0,
    0,
    0,
    0,
    8,
    2,
    4,
    10,
    3,
    0,
    0,
    12,
    14,
    19,
    1,
    2,
    2,
    4,
    4,
    4,
    10,
    15,
    6,
    4,
    356,
    1621
   ],
   "wrong": [
    1797,
    61,
    30,
    25,
    13,
    13,
    3,
    0,
    0,
    1,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    10,
    3,
    21,
    28,
    9,
    0,
    0,
    6,
    9,
    11,
    0,
    12,
    4,
    7,
    1,
    0,
    6,
    3,
    3,
    0,
    206,
    30
   ]
  },
  "patterns": [
   {
    "verdicts": "vvv",
    "n": 2154,
    "right": 1922,
    "p": 0.9902
   },
   {
    "verdicts": "vvx",
    "n": 113,
    "right": 83,
    "p": 0.7678
   },
   {
    "verdicts": "vxv",
    "n": 120,
    "right": 59,
    "p": 0.7062
   },
   {
    "verdicts": "vxx",
    "n": 58,
    "right": 32,
    "p": 0.1047
   },
   {
    "verdicts": "xvv",
    "n": 83,
    "right": 37,
    "p": 0.6894
   },
   {
    "verdicts": "xvx",
    "n": 61,
    "right": 19,
    "p": 0.0715
   },
   {
    "verdicts": "xxv",
    "n": 126,
    "right": 34,
    "p": 0.0533
   },
   {
    "verdicts": "xxx",
    "n": 1836,
    "right": 53,
    "p": 0.0025
   }
  ]
 }
]