{
"summary": {
"horizon": 20000,
"runs": 20,
"spread": {
"imdb": {
"gains": [
0.0010000000000000009,
0.008000000000000007,
0.0,
0.0,
0.0,
0.0,
0.133,
0.0,
0.019999999999999962,
0.15900000000000003
],
"worst": [
0.951,
0.965,
0.946,
0.948,
0.95,
0.92,
0.314,
0.312,
0.22599999999999998,
0.935
]
},
"civil_comments": {
"gains": [
0.04700000000000004,
0.0,
0.638,
0.0,
0.02700000000000001,
0.0,
0.02300000000000002,
0.361,
0.04799999999999999,
0.0
],
"worst": [
0.795,
0.892,
0.762,
0.135,
0.135,
0.112,
0.889,
0.569,
0.37,
0.688
]
},
"natural_qa": {
"gains": [
0.023938916834575497,
0.0021067735073739824,
0.0,
0.0,
0.011211030647736209,
0.01865392949790723,
0.0,
0.0,
0.02625015471906117,
0.0
],
"worst": [
0.42666432226406553,
0.41079285498215234,
0.3684843313150657,
0.3749096327528897,
0.39961626381204013,
0.39936099427709837,
0.04008861032005345,
0.4205717276307035,
0.2519291907869733,
0.086727784493874
]
}
},
"strategies": [
{
"strategy": "keep the default",
"within_1_point": 0.5666666666666667,
"decided": null,
"median_requests": null,
"shortfall_per_1000": 51.5720268402218
},
{
"strategy": "uniform",
"within_1_point": 0.99,
"decided": 0.30333333333333334,
"median_requests": 1800.0,
"shortfall_per_1000": 12.049560301146867
},
{
"strategy": "thompson",
"within_1_point": 0.9983333333333333,
"decided": 0.0,
"median_requests": null,
"shortfall_per_1000": 3.517393996704295
}
],
"identical": [
{
"allocation": "uniform",
"false": 8,
"runs": 600
},
{
"allocation": "thompson",
"false": 1,
"runs": 600
}
]
},
"experiments": [
{
"name": "bloom/civil_comments",
"model": "bloom",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.748,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
252
],
[
1.0,
748
]
]
},
{
"name": "expert",
"mean": 0.694,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
306
],
[
1.0,
694
]
]
},
{
"name": "i_o",
"mean": 0.75,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
250
],
[
1.0,
750
]
]
},
{
"name": "input_output",
"mean": 0.795,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
205
],
[
1.0,
795
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "bloom/imdb",
"model": "bloom",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.95,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
50
],
[
1.0,
950
]
]
},
{
"name": "expert",
"mean": 0.951,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
49
],
[
1.0,
951
]
]
},
{
"name": "i_o",
"mean": 0.919,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
81
],
[
1.0,
919
]
]
},
{
"name": "input_output",
"mean": 0.932,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
68
],
[
1.0,
932
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "bloom/natural_qa",
"model": "bloom",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.6256,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
231
],
[
0.05,
1
],
[
0.1,
12
],
[
0.15,
8
],
[
0.2,
16
],
[
0.25,
11
],
[
0.3,
15
],
[
0.35,
17
],
[
0.4,
20
],
[
0.45,
9
],
[
0.5,
71
],
[
0.55,
13
],
[
0.6,
7
],
[
0.65,
40
],
[
0.75,
8
],
[
0.8,
24
],
[
0.85,
3
],
[
0.9,
6
],
[
0.95,
2
],
[
1.0,
486
]
]
},
{
"name": "expert",
"mean": 0.6495,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
205
],
[
0.05,
2
],
[
0.1,
10
],
[
0.15,
11
],
[
0.2,
20
],
[
0.25,
8
],
[
0.3,
12
],
[
0.35,
22
],
[
0.4,
23
],
[
0.45,
8
],
[
0.5,
61
],
[
0.55,
15
],
[
0.6,
5
],
[
0.65,
43
],
[
0.75,
10
],
[
0.8,
24
],
[
0.85,
3
],
[
0.9,
8
],
[
0.95,
5
],
[
1.0,
505
]
]
},
{
"name": "i_o",
"mean": 0.6117,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
238
],
[
0.05,
1
],
[
0.1,
10
],
[
0.15,
12
],
[
0.2,
15
],
[
0.25,
11
],
[
0.3,
12
],
[
0.35,
22
],
[
0.4,
24
],
[
0.45,
8
],
[
0.5,
76
],
[
0.55,
9
],
[
0.6,
2
],
[
0.65,
51
],
[
0.7,
1
],
[
0.75,
7
],
[
0.8,
26
],
[
0.85,
5
],
[
0.9,
7
],
[
0.95,
2
],
[
1.0,
461
]
]
},
{
"name": "input_output",
"mean": 0.6421,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
208
],
[
0.05,
4
],
[
0.1,
9
],
[
0.15,
13
],
[
0.2,
19
],
[
0.25,
8
],
[
0.3,
13
],
[
0.35,
26
],
[
0.4,
26
],
[
0.45,
6
],
[
0.5,
63
],
[
0.55,
13
],
[
0.6,
4
],
[
0.65,
41
],
[
0.7,
2
],
[
0.75,
7
],
[
0.8,
21
],
[
0.85,
3
],
[
0.9,
10
],
[
0.95,
5
],
[
1.0,
499
]
]
},
{
"name": "html",
"mean": 0.2229,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
553
],
[
0.05,
3
],
[
0.1,
9
],
[
0.15,
7
],
[
0.2,
20
],
[
0.25,
12
],
[
0.3,
12
],
[
0.35,
25
],
[
0.4,
29
],
[
0.45,
6
],
[
0.5,
204
],
[
0.55,
7
],
[
0.6,
4
],
[
0.65,
70
],
[
0.75,
12
],
[
0.8,
9
],
[
0.85,
8
],
[
0.9,
5
],
[
0.95,
3
],
[
1.0,
2
]
]
}
]
},
{
"name": "glm/civil_comments",
"model": "glm",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.892,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
108
],
[
1.0,
892
]
]
},
{
"name": "expert",
"mean": 0.892,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
108
],
[
1.0,
892
]
]
},
{
"name": "i_o",
"mean": 0.892,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
108
],
[
1.0,
892
]
]
},
{
"name": "input_output",
"mean": 0.892,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
108
],
[
1.0,
892
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "glm/imdb",
"model": "glm",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.957,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
43
],
[
1.0,
957
]
]
},
{
"name": "expert",
"mean": 0.964,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
36
],
[
1.0,
964
]
]
},
{
"name": "i_o",
"mean": 0.965,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
35
],
[
1.0,
965
]
]
},
{
"name": "input_output",
"mean": 0.963,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
37
],
[
1.0,
963
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "glm/natural_qa",
"model": "glm",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.6491,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
184
],
[
0.05,
12
],
[
0.1,
12
],
[
0.15,
13
],
[
0.2,
20
],
[
0.25,
11
],
[
0.3,
17
],
[
0.35,
23
],
[
0.4,
20
],
[
0.45,
6
],
[
0.5,
66
],
[
0.55,
18
],
[
0.6,
4
],
[
0.65,
54
],
[
0.75,
8
],
[
0.8,
20
],
[
0.85,
4
],
[
0.9,
1
],
[
0.95,
2
],
[
1.0,
505
]
]
},
{
"name": "expert",
"mean": 0.6512,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
188
],
[
0.05,
7
],
[
0.1,
17
],
[
0.15,
14
],
[
0.2,
17
],
[
0.25,
11
],
[
0.3,
16
],
[
0.35,
19
],
[
0.4,
19
],
[
0.45,
7
],
[
0.5,
65
],
[
0.55,
15
],
[
0.6,
4
],
[
0.65,
56
],
[
0.75,
8
],
[
0.8,
20
],
[
0.85,
3
],
[
0.9,
2
],
[
0.95,
2
],
[
1.0,
510
]
]
},
{
"name": "i_o",
"mean": 0.6267,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
212
],
[
0.05,
11
],
[
0.1,
16
],
[
0.15,
11
],
[
0.2,
17
],
[
0.25,
9
],
[
0.3,
19
],
[
0.35,
24
],
[
0.4,
19
],
[
0.45,
8
],
[
0.5,
62
],
[
0.55,
16
],
[
0.6,
2
],
[
0.65,
50
],
[
0.75,
7
],
[
0.8,
21
],
[
0.85,
4
],
[
0.95,
1
],
[
1.0,
491
]
]
},
{
"name": "input_output",
"mean": 0.6463,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
195
],
[
0.05,
9
],
[
0.1,
17
],
[
0.15,
12
],
[
0.2,
18
],
[
0.25,
10
],
[
0.3,
16
],
[
0.35,
21
],
[
0.4,
17
],
[
0.45,
7
],
[
0.5,
60
],
[
0.55,
16
],
[
0.6,
1
],
[
0.65,
60
],
[
0.7,
1
],
[
0.75,
5
],
[
0.8,
22
],
[
0.85,
2
],
[
0.9,
2
],
[
0.95,
2
],
[
1.0,
507
]
]
},
{
"name": "html",
"mean": 0.2404,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
513
],
[
0.05,
14
],
[
0.1,
14
],
[
0.15,
11
],
[
0.2,
11
],
[
0.25,
14
],
[
0.3,
11
],
[
0.35,
22
],
[
0.4,
26
],
[
0.45,
7
],
[
0.5,
218
],
[
0.55,
10
],
[
0.6,
8
],
[
0.65,
78
],
[
0.75,
16
],
[
0.8,
9
],
[
0.85,
10
],
[
0.9,
4
],
[
0.95,
3
],
[
1.0,
1
]
]
}
]
},
{
"name": "gpt-j-6b/civil_comments",
"model": "gpt-j-6b",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.124,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
876
],
[
1.0,
124
]
]
},
{
"name": "expert",
"mean": 0.117,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
883
],
[
1.0,
117
]
]
},
{
"name": "i_o",
"mean": 0.112,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
888
],
[
1.0,
112
]
]
},
{
"name": "input_output",
"mean": 0.762,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
238
],
[
1.0,
762
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "gpt-j-6b/imdb",
"model": "gpt-j-6b",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.946,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
54
],
[
1.0,
946
]
]
},
{
"name": "expert",
"mean": 0.941,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
59
],
[
1.0,
941
]
]
},
{
"name": "i_o",
"mean": 0.938,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
62
],
[
1.0,
938
]
]
},
{
"name": "input_output",
"mean": 0.942,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
58
],
[
1.0,
942
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "gpt-j-6b/natural_qa",
"model": "gpt-j-6b",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.5697,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
265
],
[
0.05,
5
],
[
0.1,
10
],
[
0.15,
14
],
[
0.2,
29
],
[
0.25,
6
],
[
0.3,
13
],
[
0.35,
26
],
[
0.4,
20
],
[
0.45,
10
],
[
0.5,
69
],
[
0.55,
19
],
[
0.6,
5
],
[
0.65,
49
],
[
0.7,
2
],
[
0.75,
10
],
[
0.8,
13
],
[
0.85,
3
],
[
0.9,
8
],
[
0.95,
1
],
[
1.0,
423
]
]
},
{
"name": "expert",
"mean": 0.5627,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
278
],
[
0.05,
4
],
[
0.1,
5
],
[
0.15,
12
],
[
0.2,
30
],
[
0.25,
7
],
[
0.3,
13
],
[
0.35,
26
],
[
0.4,
20
],
[
0.45,
9
],
[
0.5,
70
],
[
0.55,
17
],
[
0.6,
4
],
[
0.65,
49
],
[
0.7,
1
],
[
0.75,
12
],
[
0.8,
15
],
[
0.85,
3
],
[
0.9,
9
],
[
0.95,
2
],
[
1.0,
414
]
]
},
{
"name": "i_o",
"mean": 0.5365,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
311
],
[
0.05,
2
],
[
0.1,
10
],
[
0.15,
14
],
[
0.2,
30
],
[
0.25,
7
],
[
0.3,
13
],
[
0.35,
23
],
[
0.4,
16
],
[
0.45,
6
],
[
0.5,
75
],
[
0.55,
14
],
[
0.6,
1
],
[
0.65,
42
],
[
0.7,
1
],
[
0.75,
8
],
[
0.8,
10
],
[
0.85,
1
],
[
0.9,
8
],
[
0.95,
3
],
[
1.0,
405
]
]
},
{
"name": "input_output",
"mean": 0.4986,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
368
],
[
0.05,
4
],
[
0.1,
8
],
[
0.15,
9
],
[
0.2,
19
],
[
0.25,
5
],
[
0.3,
14
],
[
0.35,
22
],
[
0.4,
13
],
[
0.45,
6
],
[
0.5,
68
],
[
0.55,
13
],
[
0.6,
3
],
[
0.65,
43
],
[
0.75,
9
],
[
0.8,
10
],
[
0.85,
2
],
[
0.9,
5
],
[
0.95,
2
],
[
1.0,
377
]
]
},
{
"name": "html",
"mean": 0.2013,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
600
],
[
0.05,
4
],
[
0.1,
9
],
[
0.15,
13
],
[
0.2,
9
],
[
0.25,
5
],
[
0.3,
7
],
[
0.35,
25
],
[
0.4,
21
],
[
0.45,
6
],
[
0.5,
193
],
[
0.55,
5
],
[
0.6,
4
],
[
0.65,
63
],
[
0.7,
1
],
[
0.75,
14
],
[
0.8,
6
],
[
0.85,
5
],
[
0.9,
4
],
[
0.95,
1
],
[
1.0,
5
]
]
}
]
},
{
"name": "gpt-neox-20b/civil_comments",
"model": "gpt-neox-20b",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.135,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
865
],
[
1.0,
135
]
]
},
{
"name": "expert",
"mean": 0.123,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
877
],
[
1.0,
123
]
]
},
{
"name": "i_o",
"mean": 0.113,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
887
],
[
1.0,
113
]
]
},
{
"name": "input_output",
"mean": 0.113,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
887
],
[
1.0,
113
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "gpt-neox-20b/imdb",
"model": "gpt-neox-20b",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.948,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
52
],
[
1.0,
948
]
]
},
{
"name": "expert",
"mean": 0.943,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
57
],
[
1.0,
943
]
]
},
{
"name": "i_o",
"mean": 0.927,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
73
],
[
1.0,
927
]
]
},
{
"name": "input_output",
"mean": 0.936,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
64
],
[
1.0,
936
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "gpt-neox-20b/natural_qa",
"model": "gpt-neox-20b",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.5995,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
251
],
[
0.05,
2
],
[
0.1,
12
],
[
0.15,
13
],
[
0.2,
20
],
[
0.25,
12
],
[
0.3,
12
],
[
0.35,
23
],
[
0.4,
20
],
[
0.45,
8
],
[
0.5,
66
],
[
0.55,
15
],
[
0.6,
3
],
[
0.65,
37
],
[
0.7,
1
],
[
0.75,
8
],
[
0.8,
19
],
[
0.85,
5
],
[
0.9,
10
],
[
0.95,
2
],
[
1.0,
461
]
]
},
{
"name": "expert",
"mean": 0.599,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
258
],
[
0.05,
4
],
[
0.1,
10
],
[
0.15,
14
],
[
0.2,
17
],
[
0.25,
11
],
[
0.3,
13
],
[
0.35,
22
],
[
0.4,
19
],
[
0.45,
9
],
[
0.5,
54
],
[
0.55,
14
],
[
0.6,
1
],
[
0.65,
46
],
[
0.7,
2
],
[
0.75,
7
],
[
0.8,
17
],
[
0.85,
7
],
[
0.9,
10
],
[
0.95,
1
],
[
1.0,
464
]
]
},
{
"name": "i_o",
"mean": 0.5815,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
275
],
[
0.05,
3
],
[
0.1,
11
],
[
0.15,
13
],
[
0.2,
14
],
[
0.25,
11
],
[
0.3,
15
],
[
0.35,
18
],
[
0.4,
18
],
[
0.45,
6
],
[
0.5,
70
],
[
0.55,
17
],
[
0.6,
1
],
[
0.65,
41
],
[
0.75,
8
],
[
0.8,
18
],
[
0.85,
5
],
[
0.9,
8
],
[
0.95,
2
],
[
1.0,
446
]
]
},
{
"name": "input_output",
"mean": 0.5761,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
277
],
[
0.05,
4
],
[
0.1,
7
],
[
0.15,
10
],
[
0.2,
22
],
[
0.25,
9
],
[
0.3,
22
],
[
0.35,
17
],
[
0.4,
22
],
[
0.45,
7
],
[
0.5,
62
],
[
0.55,
14
],
[
0.6,
4
],
[
0.65,
41
],
[
0.75,
10
],
[
0.8,
18
],
[
0.85,
7
],
[
0.9,
5
],
[
0.95,
1
],
[
1.0,
441
]
]
},
{
"name": "html",
"mean": 0.2246,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
554
],
[
0.05,
6
],
[
0.1,
5
],
[
0.15,
10
],
[
0.2,
16
],
[
0.25,
7
],
[
0.3,
15
],
[
0.35,
27
],
[
0.4,
26
],
[
0.45,
10
],
[
0.5,
202
],
[
0.55,
6
],
[
0.6,
2
],
[
0.65,
64
],
[
0.75,
20
],
[
0.8,
12
],
[
0.85,
4
],
[
0.9,
10
],
[
0.95,
2
],
[
1.0,
2
]
]
}
]
},
{
"name": "opt-175b/civil_comments",
"model": "opt-175b",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.108,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
892
],
[
1.0,
108
]
]
},
{
"name": "expert",
"mean": 0.108,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
892
],
[
1.0,
108
]
]
},
{
"name": "i_o",
"mean": 0.135,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
865
],
[
1.0,
135
]
]
},
{
"name": "input_output",
"mean": 0.117,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
883
],
[
1.0,
117
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "opt-175b/imdb",
"model": "opt-175b",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.95,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
50
],
[
1.0,
950
]
]
},
{
"name": "expert",
"mean": 0.948,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
52
],
[
1.0,
948
]
]
},
{
"name": "i_o",
"mean": 0.925,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
75
],
[
1.0,
925
]
]
},
{
"name": "input_output",
"mean": 0.949,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
51
],
[
1.0,
949
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "opt-175b/natural_qa",
"model": "opt-175b",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.6071,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
242
],
[
0.05,
1
],
[
0.1,
8
],
[
0.15,
16
],
[
0.2,
21
],
[
0.25,
11
],
[
0.3,
14
],
[
0.35,
17
],
[
0.4,
22
],
[
0.45,
5
],
[
0.5,
70
],
[
0.55,
13
],
[
0.6,
5
],
[
0.65,
44
],
[
0.7,
1
],
[
0.75,
10
],
[
0.8,
26
],
[
0.85,
6
],
[
0.9,
7
],
[
0.95,
5
],
[
1.0,
456
]
]
},
{
"name": "expert",
"mean": 0.6183,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
231
],
[
0.05,
1
],
[
0.1,
8
],
[
0.15,
15
],
[
0.2,
21
],
[
0.25,
10
],
[
0.3,
14
],
[
0.35,
20
],
[
0.4,
22
],
[
0.45,
3
],
[
0.5,
74
],
[
0.55,
12
],
[
0.6,
4
],
[
0.65,
42
],
[
0.7,
1
],
[
0.75,
11
],
[
0.8,
25
],
[
0.85,
5
],
[
0.9,
7
],
[
0.95,
4
],
[
1.0,
470
]
]
},
{
"name": "i_o",
"mean": 0.604,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
247
],
[
0.05,
3
],
[
0.1,
6
],
[
0.15,
10
],
[
0.2,
18
],
[
0.25,
11
],
[
0.3,
17
],
[
0.35,
25
],
[
0.4,
29
],
[
0.45,
2
],
[
0.5,
62
],
[
0.55,
11
],
[
0.6,
2
],
[
0.65,
51
],
[
0.75,
7
],
[
0.8,
32
],
[
0.85,
2
],
[
0.9,
5
],
[
0.95,
4
],
[
1.0,
456
]
]
},
{
"name": "input_output",
"mean": 0.6042,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
241
],
[
0.05,
3
],
[
0.1,
10
],
[
0.15,
13
],
[
0.2,
23
],
[
0.25,
9
],
[
0.3,
16
],
[
0.35,
23
],
[
0.4,
29
],
[
0.45,
5
],
[
0.5,
61
],
[
0.55,
11
],
[
0.6,
2
],
[
0.65,
50
],
[
0.75,
7
],
[
0.8,
21
],
[
0.85,
3
],
[
0.9,
6
],
[
0.95,
5
],
[
1.0,
462
]
]
},
{
"name": "html",
"mean": 0.2187,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
555
],
[
0.05,
3
],
[
0.1,
16
],
[
0.15,
9
],
[
0.2,
16
],
[
0.25,
12
],
[
0.3,
15
],
[
0.35,
27
],
[
0.4,
29
],
[
0.45,
11
],
[
0.5,
187
],
[
0.55,
8
],
[
0.6,
5
],
[
0.65,
64
],
[
0.75,
11
],
[
0.8,
13
],
[
0.85,
6
],
[
0.9,
10
],
[
1.0,
3
]
]
}
]
},
{
"name": "opt-66b/civil_comments",
"model": "opt-66b",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.112,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
888
],
[
1.0,
112
]
]
},
{
"name": "expert",
"mean": 0.0,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
1000
]
]
},
{
"name": "i_o",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
1000
]
]
},
{
"name": "input_output",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
1000
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "opt-66b/imdb",
"model": "opt-66b",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.92,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
80
],
[
1.0,
920
]
]
},
{
"name": "expert",
"mean": 0.0,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
1000
]
]
},
{
"name": "i_o",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
1000
]
]
},
{
"name": "input_output",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
1000
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "opt-66b/natural_qa",
"model": "opt-66b",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.5905,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
250
],
[
0.05,
1
],
[
0.1,
14
],
[
0.15,
12
],
[
0.2,
14
],
[
0.25,
16
],
[
0.3,
17
],
[
0.35,
21
],
[
0.4,
21
],
[
0.45,
8
],
[
0.5,
76
],
[
0.55,
17
],
[
0.6,
6
],
[
0.65,
43
],
[
0.7,
2
],
[
0.75,
8
],
[
0.8,
18
],
[
0.85,
7
],
[
0.9,
3
],
[
0.95,
3
],
[
1.0,
443
]
]
},
{
"name": "expert",
"mean": 0.6033,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
240
],
[
0.05,
3
],
[
0.1,
11
],
[
0.15,
10
],
[
0.2,
14
],
[
0.25,
16
],
[
0.3,
18
],
[
0.35,
23
],
[
0.4,
20
],
[
0.45,
8
],
[
0.5,
72
],
[
0.55,
17
],
[
0.6,
4
],
[
0.65,
45
],
[
0.7,
1
],
[
0.75,
8
],
[
0.8,
20
],
[
0.85,
8
],
[
0.9,
4
],
[
0.95,
2
],
[
1.0,
456
]
]
},
{
"name": "i_o",
"mean": 0.582,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
263
],
[
0.05,
3
],
[
0.1,
11
],
[
0.15,
9
],
[
0.2,
16
],
[
0.25,
15
],
[
0.3,
13
],
[
0.35,
27
],
[
0.4,
24
],
[
0.45,
4
],
[
0.5,
73
],
[
0.55,
12
],
[
0.65,
51
],
[
0.75,
6
],
[
0.8,
24
],
[
0.85,
8
],
[
0.9,
7
],
[
0.95,
1
],
[
1.0,
433
]
]
},
{
"name": "input_output",
"mean": 0.6091,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
243
],
[
0.05,
6
],
[
0.1,
7
],
[
0.15,
7
],
[
0.2,
17
],
[
0.25,
6
],
[
0.3,
18
],
[
0.35,
20
],
[
0.4,
22
],
[
0.45,
5
],
[
0.5,
67
],
[
0.55,
16
],
[
0.6,
5
],
[
0.65,
57
],
[
0.75,
11
],
[
0.8,
19
],
[
0.85,
7
],
[
0.9,
9
],
[
0.95,
2
],
[
1.0,
456
]
]
},
{
"name": "html",
"mean": 0.2098,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
570
],
[
0.05,
7
],
[
0.1,
15
],
[
0.15,
7
],
[
0.2,
9
],
[
0.25,
12
],
[
0.3,
18
],
[
0.35,
25
],
[
0.4,
27
],
[
0.45,
9
],
[
0.5,
190
],
[
0.55,
12
],
[
0.6,
1
],
[
0.65,
60
],
[
0.75,
12
],
[
0.8,
13
],
[
0.85,
4
],
[
0.9,
5
],
[
0.95,
1
],
[
1.0,
3
]
]
}
]
},
{
"name": "t0pp/civil_comments",
"model": "t0pp",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.866,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
134
],
[
1.0,
866
]
]
},
{
"name": "expert",
"mean": 0.069,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
931
],
[
1.0,
69
]
]
},
{
"name": "i_o",
"mean": 0.889,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
111
],
[
1.0,
889
]
]
},
{
"name": "input_output",
"mean": 0.866,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
134
],
[
1.0,
866
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "t0pp/imdb",
"model": "t0pp",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.181,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
819
],
[
1.0,
181
]
]
},
{
"name": "expert",
"mean": 0.172,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
828
],
[
1.0,
172
]
]
},
{
"name": "i_o",
"mean": 0.205,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
795
],
[
1.0,
205
]
]
},
{
"name": "input_output",
"mean": 0.314,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
686
],
[
1.0,
314
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "t0pp/natural_qa",
"model": "t0pp",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.2026,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
373
],
[
0.05,
71
],
[
0.1,
81
],
[
0.15,
54
],
[
0.2,
48
],
[
0.25,
30
],
[
0.3,
46
],
[
0.35,
41
],
[
0.4,
75
],
[
0.45,
18
],
[
0.5,
55
],
[
0.55,
31
],
[
0.6,
12
],
[
0.65,
35
],
[
0.7,
5
],
[
0.75,
11
],
[
0.8,
5
],
[
0.85,
7
],
[
0.9,
1
],
[
0.95,
1
]
]
},
{
"name": "expert",
"mean": 0.2008,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
377
],
[
0.05,
71
],
[
0.1,
80
],
[
0.15,
52
],
[
0.2,
50
],
[
0.25,
30
],
[
0.3,
46
],
[
0.35,
40
],
[
0.4,
77
],
[
0.45,
18
],
[
0.5,
53
],
[
0.55,
31
],
[
0.6,
12
],
[
0.65,
33
],
[
0.7,
5
],
[
0.75,
11
],
[
0.8,
5
],
[
0.85,
7
],
[
0.9,
1
],
[
0.95,
1
]
]
},
{
"name": "i_o",
"mean": 0.173,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
470
],
[
0.05,
65
],
[
0.1,
57
],
[
0.15,
35
],
[
0.2,
44
],
[
0.25,
21
],
[
0.3,
42
],
[
0.35,
42
],
[
0.4,
61
],
[
0.45,
20
],
[
0.5,
62
],
[
0.55,
28
],
[
0.6,
5
],
[
0.65,
32
],
[
0.7,
3
],
[
0.75,
4
],
[
0.8,
5
],
[
0.85,
2
],
[
0.9,
1
],
[
0.95,
1
]
]
},
{
"name": "input_output",
"mean": 0.1886,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
426
],
[
0.05,
61
],
[
0.1,
64
],
[
0.15,
47
],
[
0.2,
39
],
[
0.25,
24
],
[
0.3,
54
],
[
0.35,
39
],
[
0.4,
73
],
[
0.45,
27
],
[
0.5,
62
],
[
0.55,
21
],
[
0.6,
5
],
[
0.65,
36
],
[
0.7,
5
],
[
0.75,
6
],
[
0.8,
4
],
[
0.85,
4
],
[
0.9,
2
],
[
0.95,
1
]
]
},
{
"name": "html",
"mean": 0.1625,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
474
],
[
0.05,
72
],
[
0.1,
56
],
[
0.15,
43
],
[
0.2,
36
],
[
0.25,
25
],
[
0.3,
54
],
[
0.35,
38
],
[
0.4,
68
],
[
0.45,
23
],
[
0.5,
35
],
[
0.55,
32
],
[
0.6,
6
],
[
0.65,
20
],
[
0.7,
2
],
[
0.75,
6
],
[
0.8,
4
],
[
0.85,
6
]
]
}
]
},
{
"name": "t5-11b/civil_comments",
"model": "t5-11b",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.208,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
792
],
[
1.0,
208
]
]
},
{
"name": "expert",
"mean": 0.569,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
431
],
[
1.0,
569
]
]
},
{
"name": "i_o",
"mean": 0.356,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
644
],
[
1.0,
356
]
]
},
{
"name": "input_output",
"mean": 0.272,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
728
],
[
1.0,
272
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "t5-11b/imdb",
"model": "t5-11b",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.321,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
679
],
[
1.0,
321
]
]
},
{
"name": "expert",
"mean": 0.049,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
951
],
[
1.0,
49
]
]
},
{
"name": "i_o",
"mean": 0.251,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
749
],
[
1.0,
251
]
]
},
{
"name": "input_output",
"mean": 0.248,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
752
],
[
1.0,
248
]
]
},
{
"name": "html",
"mean": 0.009,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
991
],
[
1.0,
9
]
]
}
]
},
{
"name": "t5-11b/natural_qa",
"model": "t5-11b",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.5656,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
261
],
[
0.05,
4
],
[
0.1,
14
],
[
0.15,
21
],
[
0.2,
20
],
[
0.25,
16
],
[
0.3,
21
],
[
0.35,
16
],
[
0.4,
24
],
[
0.45,
13
],
[
0.5,
56
],
[
0.55,
23
],
[
0.6,
3
],
[
0.65,
49
],
[
0.7,
2
],
[
0.75,
10
],
[
0.8,
22
],
[
0.85,
6
],
[
0.9,
5
],
[
1.0,
414
]
]
},
{
"name": "expert",
"mean": 0.5302,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
276
],
[
0.05,
15
],
[
0.1,
17
],
[
0.15,
23
],
[
0.2,
21
],
[
0.25,
15
],
[
0.3,
23
],
[
0.35,
23
],
[
0.4,
27
],
[
0.45,
12
],
[
0.5,
50
],
[
0.55,
24
],
[
0.6,
4
],
[
0.65,
52
],
[
0.7,
1
],
[
0.75,
12
],
[
0.8,
20
],
[
0.85,
3
],
[
0.9,
6
],
[
0.95,
1
],
[
1.0,
375
]
]
},
{
"name": "i_o",
"mean": 0.5503,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
287
],
[
0.05,
6
],
[
0.1,
15
],
[
0.15,
15
],
[
0.2,
22
],
[
0.25,
11
],
[
0.3,
23
],
[
0.35,
11
],
[
0.4,
25
],
[
0.45,
7
],
[
0.5,
61
],
[
0.55,
19
],
[
0.6,
3
],
[
0.65,
45
],
[
0.7,
2
],
[
0.75,
9
],
[
0.8,
19
],
[
0.85,
4
],
[
0.9,
5
],
[
0.95,
1
],
[
1.0,
410
]
]
},
{
"name": "input_output",
"mean": 0.5615,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
276
],
[
0.05,
8
],
[
0.1,
17
],
[
0.15,
8
],
[
0.2,
24
],
[
0.25,
12
],
[
0.3,
22
],
[
0.35,
11
],
[
0.4,
22
],
[
0.45,
13
],
[
0.5,
58
],
[
0.55,
17
],
[
0.6,
2
],
[
0.65,
54
],
[
0.7,
1
],
[
0.75,
6
],
[
0.8,
17
],
[
0.85,
5
],
[
0.9,
4
],
[
1.0,
423
]
]
},
{
"name": "html",
"mean": 0.145,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
674
],
[
0.05,
7
],
[
0.1,
26
],
[
0.15,
11
],
[
0.2,
11
],
[
0.25,
7
],
[
0.3,
9
],
[
0.35,
9
],
[
0.4,
17
],
[
0.5,
173
],
[
0.55,
4
],
[
0.6,
2
],
[
0.65,
42
],
[
0.7,
1
],
[
0.75,
5
],
[
0.8,
2
]
]
}
]
},
{
"name": "ul2/civil_comments",
"model": "ul2",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.322,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
678
],
[
1.0,
322
]
]
},
{
"name": "expert",
"mean": 0.256,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
744
],
[
1.0,
256
]
]
},
{
"name": "i_o",
"mean": 0.37,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
630
],
[
1.0,
370
]
]
},
{
"name": "input_output",
"mean": 0.286,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
714
],
[
1.0,
286
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "ul2/imdb",
"model": "ul2",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.324,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
676
],
[
1.0,
324
]
]
},
{
"name": "expert",
"mean": 0.254,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
746
],
[
1.0,
254
]
]
},
{
"name": "i_o",
"mean": 0.329,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
671
],
[
1.0,
329
]
]
},
{
"name": "input_output",
"mean": 0.344,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
656
],
[
1.0,
344
]
]
},
{
"name": "html",
"mean": 0.118,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
882
],
[
1.0,
118
]
]
}
]
},
{
"name": "ul2/natural_qa",
"model": "ul2",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.4203,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
344
],
[
0.05,
60
],
[
0.1,
56
],
[
0.15,
28
],
[
0.2,
12
],
[
0.25,
11
],
[
0.3,
14
],
[
0.35,
25
],
[
0.4,
16
],
[
0.45,
7
],
[
0.5,
48
],
[
0.55,
8
],
[
0.6,
7
],
[
0.65,
32
],
[
0.7,
1
],
[
0.75,
5
],
[
0.8,
19
],
[
0.85,
4
],
[
0.9,
3
],
[
0.95,
1
],
[
1.0,
299
]
]
},
{
"name": "expert",
"mean": 0.4465,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
328
],
[
0.05,
52
],
[
0.1,
45
],
[
0.15,
25
],
[
0.2,
19
],
[
0.25,
16
],
[
0.3,
14
],
[
0.35,
15
],
[
0.4,
16
],
[
0.45,
9
],
[
0.5,
52
],
[
0.55,
9
],
[
0.6,
5
],
[
0.65,
35
],
[
0.7,
3
],
[
0.75,
9
],
[
0.8,
23
],
[
0.85,
5
],
[
0.9,
5
],
[
0.95,
1
],
[
1.0,
314
]
]
},
{
"name": "i_o",
"mean": 0.2161,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
675
],
[
0.05,
25
],
[
0.1,
32
],
[
0.15,
6
],
[
0.2,
8
],
[
0.25,
7
],
[
0.3,
4
],
[
0.35,
5
],
[
0.4,
11
],
[
0.45,
4
],
[
0.5,
26
],
[
0.55,
6
],
[
0.65,
19
],
[
0.7,
1
],
[
0.75,
2
],
[
0.8,
8
],
[
0.85,
1
],
[
0.95,
2
],
[
1.0,
158
]
]
},
{
"name": "input_output",
"mean": 0.3893,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
349
],
[
0.05,
80
],
[
0.1,
66
],
[
0.15,
21
],
[
0.2,
20
],
[
0.25,
15
],
[
0.3,
12
],
[
0.35,
18
],
[
0.4,
15
],
[
0.45,
9
],
[
0.5,
47
],
[
0.55,
12
],
[
0.6,
5
],
[
0.65,
35
],
[
0.7,
2
],
[
0.75,
5
],
[
0.8,
12
],
[
0.85,
2
],
[
0.9,
1
],
[
1.0,
274
]
]
},
{
"name": "html",
"mean": 0.1946,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
584
],
[
0.05,
14
],
[
0.1,
20
],
[
0.15,
14
],
[
0.2,
14
],
[
0.25,
6
],
[
0.3,
14
],
[
0.35,
16
],
[
0.4,
28
],
[
0.45,
9
],
[
0.5,
181
],
[
0.55,
11
],
[
0.6,
2
],
[
0.65,
55
],
[
0.75,
9
],
[
0.8,
11
],
[
0.85,
6
],
[
0.9,
6
]
]
}
]
},
{
"name": "yalm/civil_comments",
"model": "yalm",
"task": "civil_comments",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.688,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
312
],
[
1.0,
688
]
]
},
{
"name": "expert",
"mean": 0.492,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
508
],
[
1.0,
492
]
]
},
{
"name": "i_o",
"mean": 0.475,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
525
],
[
1.0,
475
]
]
},
{
"name": "input_output",
"mean": 0.635,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
365
],
[
1.0,
635
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "yalm/imdb",
"model": "yalm",
"task": "imdb",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.776,
"prompt": {
"instructions": "",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
224
],
[
1.0,
776
]
]
},
{
"name": "expert",
"mean": 0.935,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "Passage: ",
"input_suffix": "\n",
"output_prefix": "Sentiment: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
65
],
[
1.0,
935
]
]
},
{
"name": "i_o",
"mean": 0.929,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
71
],
[
1.0,
929
]
]
},
{
"name": "input_output",
"mean": 0.934,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
66
],
[
1.0,
934
]
]
},
{
"name": "html",
"mean": 0.0,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
1000
]
]
}
]
},
{
"name": "yalm/natural_qa",
"model": "yalm",
"task": "natural_qa",
"questions": 1000,
"variants": [
{
"name": "default",
"mean": 0.2249,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
548
],
[
0.05,
11
],
[
0.1,
29
],
[
0.15,
31
],
[
0.2,
38
],
[
0.25,
33
],
[
0.3,
25
],
[
0.35,
24
],
[
0.4,
31
],
[
0.45,
10
],
[
0.5,
48
],
[
0.55,
21
],
[
0.6,
3
],
[
0.65,
39
],
[
0.7,
1
],
[
0.75,
3
],
[
0.8,
13
],
[
0.85,
3
],
[
0.9,
1
],
[
1.0,
88
]
]
},
{
"name": "expert",
"mean": 0.2245,
"prompt": {
"instructions": "I am an expert AI assistant who is here to help you with the following. ",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "Answer: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
558
],
[
0.05,
12
],
[
0.1,
23
],
[
0.15,
30
],
[
0.2,
40
],
[
0.25,
27
],
[
0.3,
26
],
[
0.35,
28
],
[
0.4,
28
],
[
0.45,
9
],
[
0.5,
43
],
[
0.55,
22
],
[
0.6,
3
],
[
0.65,
41
],
[
0.7,
1
],
[
0.75,
2
],
[
0.8,
10
],
[
0.85,
3
],
[
1.0,
94
]
]
},
{
"name": "i_o",
"mean": 0.2148,
"prompt": {
"instructions": "",
"input_prefix": "I: ",
"input_suffix": "\n",
"output_prefix": "O: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
587
],
[
0.05,
11
],
[
0.1,
20
],
[
0.15,
27
],
[
0.2,
28
],
[
0.25,
36
],
[
0.3,
21
],
[
0.35,
15
],
[
0.4,
35
],
[
0.45,
13
],
[
0.5,
37
],
[
0.55,
17
],
[
0.6,
4
],
[
0.65,
42
],
[
0.75,
4
],
[
0.8,
9
],
[
0.85,
1
],
[
1.0,
93
]
]
},
{
"name": "input_output",
"mean": 0.1381,
"prompt": {
"instructions": "",
"input_prefix": "Input: ",
"input_suffix": "\n",
"output_prefix": "Output: ",
"output_suffix": "\n"
},
"hist": [
[
0.0,
740
],
[
0.05,
2
],
[
0.1,
13
],
[
0.15,
12
],
[
0.2,
22
],
[
0.25,
13
],
[
0.3,
17
],
[
0.35,
17
],
[
0.4,
24
],
[
0.45,
5
],
[
0.5,
34
],
[
0.55,
6
],
[
0.6,
1
],
[
0.65,
25
],
[
0.75,
1
],
[
0.8,
9
],
[
0.9,
1
],
[
1.0,
58
]
]
},
{
"name": "html",
"mean": 0.1391,
"prompt": {
"instructions": "",
"input_prefix": "",
"input_suffix": "\n",
"output_prefix": "\n"
},
"hist": [
[
0.0,
641
],
[
0.05,
6
],
[
0.1,
28
],
[
0.15,
37
],
[
0.2,
46
],
[
0.25,
24
],
[
0.3,
29
],
[
0.35,
30
],
[
0.4,
28
],
[
0.45,
19
],
[
0.5,
37
],
[
0.55,
12
],
[
0.65,
26
],
[
0.7,
1
],
[
0.75,
1
],
[
0.8,
11
],
[
0.85,
1
],
[
0.9,
1
],
[
1.0,
22
]
]
}
]
}
]
}