| { |
| "STSBenchmark": { |
| "train": { |
| "pearson": [ |
| 0.700217766857111, |
| 0.0 |
| ], |
| "spearman": [ |
| 0.694596243514894, |
| 0.0 |
| ], |
| "nsamples": 5749 |
| }, |
| "dev": { |
| "pearson": [ |
| 0.7188725565702827, |
| 7.642460243707613e-239 |
| ], |
| "spearman": [ |
| 0.7430861317995934, |
| 1.3229042265692801e-263 |
| ], |
| "nsamples": 1500 |
| }, |
| "test": { |
| "pearson": [ |
| 0.6697149917661123, |
| 3.5510311886663796e-180 |
| ], |
| "spearman": [ |
| 0.7028355722100523, |
| 6.396511507534624e-206 |
| ], |
| "nsamples": 1379 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7003515412165919, |
| "mean": 0.6962684383978353, |
| "wmean": 0.698585738312752 |
| }, |
| "spearman": { |
| "all": 0.708944371429387, |
| "mean": 0.7135059825081799, |
| "wmean": 0.7043432146203266 |
| } |
| } |
| }, |
| "SICKRelatedness": { |
| "train": { |
| "pearson": [ |
| 0.7900787687078513, |
| 0.0 |
| ], |
| "spearman": [ |
| 0.7162798220982042, |
| 0.0 |
| ], |
| "nsamples": 4500 |
| }, |
| "dev": { |
| "pearson": [ |
| 0.7941456305052318, |
| 8.704175570070149e-110 |
| ], |
| "spearman": [ |
| 0.7323591663810709, |
| 3.70700656086251e-85 |
| ], |
| "nsamples": 500 |
| }, |
| "test": { |
| "pearson": [ |
| 0.7827901659487767, |
| 0.0 |
| ], |
| "spearman": [ |
| 0.7066436772359563, |
| 0.0 |
| ], |
| "nsamples": 4927 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7866207722959027, |
| "mean": 0.7890048550539532, |
| "wmean": 0.7866661047715896 |
| }, |
| "spearman": { |
| "all": 0.7122486034563766, |
| "mean": 0.7184275552384105, |
| "wmean": 0.7123070595722788 |
| } |
| } |
| }, |
| "MR": { |
| "devacc": 80.75, |
| "acc": 80.52, |
| "ndev": 10662, |
| "ntest": 10662 |
| }, |
| "CR": { |
| "devacc": 87.02, |
| "acc": 86.41, |
| "ndev": 3775, |
| "ntest": 3775 |
| }, |
| "SUBJ": { |
| "devacc": 95.02, |
| "acc": 94.49, |
| "ndev": 10000, |
| "ntest": 10000 |
| }, |
| "MPQA": { |
| "devacc": 88.67, |
| "acc": 88.53, |
| "ndev": 10606, |
| "ntest": 10606 |
| }, |
| "SST2": { |
| "devacc": 82.8, |
| "acc": 85.89, |
| "ndev": 872, |
| "ntest": 1821 |
| }, |
| "TREC": { |
| "devacc": 78.63, |
| "acc": 88.6, |
| "ndev": 5452, |
| "ntest": 500 |
| }, |
| "MRPC": { |
| "devacc": 70.17, |
| "acc": 62.55, |
| "f1": 68.4, |
| "ndev": 4076, |
| "ntest": 1725 |
| }, |
| "STS12": { |
| "MSRpar": { |
| "pearson": [ |
| 0.3279591431665635, |
| 2.8934198828902823e-20 |
| ], |
| "spearman": [ |
| 0.37151182527291027, |
| 5.848988168730171e-26 |
| ], |
| "nsamples": 750 |
| }, |
| "MSRvid": { |
| "pearson": [ |
| 0.8600914351071427, |
| 7.615361092408985e-221 |
| ], |
| "spearman": [ |
| 0.8710479889820992, |
| 3.841468226701095e-233 |
| ], |
| "nsamples": 750 |
| }, |
| "SMTeuroparl": { |
| "pearson": [ |
| 0.49106075103459146, |
| 3.1227262696721795e-29 |
| ], |
| "spearman": [ |
| 0.5976234583132141, |
| 8.832799021137483e-46 |
| ], |
| "nsamples": 459 |
| }, |
| "surprise.OnWN": { |
| "pearson": [ |
| 0.6903453132676263, |
| 2.993644573897565e-107 |
| ], |
| "spearman": [ |
| 0.6522049690471122, |
| 4.576466079694001e-92 |
| ], |
| "nsamples": 750 |
| }, |
| "surprise.SMTnews": { |
| "pearson": [ |
| 0.5747570138816134, |
| 1.8705693938405458e-36 |
| ], |
| "spearman": [ |
| 0.6161895683302498, |
| 4.402482065232814e-43 |
| ], |
| "nsamples": 399 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.6532360047609772, |
| "mean": 0.5888427312915075, |
| "wmean": 0.5995887554438998 |
| }, |
| "spearman": { |
| "all": 0.6520942930121344, |
| "mean": 0.6217155619891171, |
| "wmean": 0.6245953644163854 |
| } |
| } |
| }, |
| "STS13": { |
| "FNWN": { |
| "pearson": [ |
| 0.5455691661756268, |
| 4.746052411411139e-16 |
| ], |
| "spearman": [ |
| 0.5552285672079286, |
| 1.1207604310221007e-16 |
| ], |
| "nsamples": 189 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.7376364893975698, |
| 1.0257488464328715e-129 |
| ], |
| "spearman": [ |
| 0.7584758247313932, |
| 3.1099800660504727e-141 |
| ], |
| "nsamples": 750 |
| }, |
| "OnWN": { |
| "pearson": [ |
| 0.8062599488981587, |
| 1.4758922804276925e-129 |
| ], |
| "spearman": [ |
| 0.791620979725446, |
| 1.0741113505197788e-121 |
| ], |
| "nsamples": 561 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7161952323138701, |
| "mean": 0.6964885348237851, |
| "wmean": 0.7391011805248252 |
| }, |
| "spearman": { |
| "all": 0.7489808796663362, |
| "mean": 0.701775123888256, |
| "wmean": 0.7452629582512124 |
| } |
| } |
| }, |
| "STS14": { |
| "deft-forum": { |
| "pearson": [ |
| 0.4539905158613596, |
| 2.8973755391032382e-24 |
| ], |
| "spearman": [ |
| 0.4539149626420418, |
| 2.954477747813041e-24 |
| ], |
| "nsamples": 450 |
| }, |
| "deft-news": { |
| "pearson": [ |
| 0.6909565157190697, |
| 6.731301106701071e-44 |
| ], |
| "spearman": [ |
| 0.654918374815774, |
| 3.9304042018460903e-38 |
| ], |
| "nsamples": 300 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.7163249206179311, |
| 5.064372905060631e-119 |
| ], |
| "spearman": [ |
| 0.6890244755927888, |
| 1.1000588945875164e-106 |
| ], |
| "nsamples": 750 |
| }, |
| "images": { |
| "pearson": [ |
| 0.8341976013585205, |
| 1.581737521468821e-195 |
| ], |
| "spearman": [ |
| 0.8009087778380325, |
| 9.127029827171149e-169 |
| ], |
| "nsamples": 750 |
| }, |
| "OnWN": { |
| "pearson": [ |
| 0.8416372752913062, |
| 2.488984870540694e-202 |
| ], |
| "spearman": [ |
| 0.835891574935076, |
| 4.7883053518051054e-197 |
| ], |
| "nsamples": 750 |
| }, |
| "tweet-news": { |
| "pearson": [ |
| 0.5893351545919092, |
| 2.4679056644216343e-71 |
| ], |
| "spearman": [ |
| 0.5523429250762462, |
| 4.020802928899476e-61 |
| ], |
| "nsamples": 750 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.6458742493245161, |
| "mean": 0.6877403305733494, |
| "wmean": 0.7060543735328222 |
| }, |
| "spearman": { |
| "all": 0.6214541243526541, |
| "mean": 0.6645001818166599, |
| "wmean": 0.6824968161907357 |
| } |
| } |
| }, |
| "STS15": { |
| "answers-forums": { |
| "pearson": [ |
| 0.5843871869461419, |
| 1.0151032193286744e-35 |
| ], |
| "spearman": [ |
| 0.6084659043667563, |
| 2.3820444508668996e-39 |
| ], |
| "nsamples": 375 |
| }, |
| "answers-students": { |
| "pearson": [ |
| 0.6152132265657764, |
| 2.6617778013870566e-79 |
| ], |
| "spearman": [ |
| 0.6302561398409405, |
| 2.773969605891675e-84 |
| ], |
| "nsamples": 750 |
| }, |
| "belief": { |
| "pearson": [ |
| 0.6392710473707196, |
| 1.799179528686185e-44 |
| ], |
| "spearman": [ |
| 0.686875976718544, |
| 1.2015551469859737e-53 |
| ], |
| "nsamples": 375 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.7734487150038147, |
| 2.927357789667028e-150 |
| ], |
| "spearman": [ |
| 0.7896018531226366, |
| 8.23498473346381e-161 |
| ], |
| "nsamples": 750 |
| }, |
| "images": { |
| "pearson": [ |
| 0.864344989852659, |
| 1.7210495814828955e-225 |
| ], |
| "spearman": [ |
| 0.8824596984841369, |
| 3.2828816211446655e-247 |
| ], |
| "nsamples": 750 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7203379074291151, |
| "mean": 0.6953330331478222, |
| "wmean": 0.7162090121451702 |
| }, |
| "spearman": { |
| "all": 0.7497940183329284, |
| "mean": 0.7195319145066028, |
| "wmean": 0.7374971579975911 |
| } |
| } |
| }, |
| "STS16": { |
| "answer-answer": { |
| "pearson": [ |
| 0.6368696070010881, |
| 2.678858858103159e-30 |
| ], |
| "spearman": [ |
| 0.6921912704278468, |
| 1.4681538672067537e-37 |
| ], |
| "nsamples": 254 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.7289776103151235, |
| 1.5277023512508603e-42 |
| ], |
| "spearman": [ |
| 0.7621038994295888, |
| 1.551008215103711e-48 |
| ], |
| "nsamples": 249 |
| }, |
| "plagiarism": { |
| "pearson": [ |
| 0.7702063765670754, |
| 2.003814813962993e-46 |
| ], |
| "spearman": [ |
| 0.7911510680474892, |
| 1.3837761303567582e-50 |
| ], |
| "nsamples": 230 |
| }, |
| "postediting": { |
| "pearson": [ |
| 0.719837823485291, |
| 3.0286294416646686e-40 |
| ], |
| "spearman": [ |
| 0.7566461459669797, |
| 1.4825051882045699e-46 |
| ], |
| "nsamples": 244 |
| }, |
| "question-question": { |
| "pearson": [ |
| 0.7625392869659763, |
| 5.114455019237254e-41 |
| ], |
| "spearman": [ |
| 0.7800407128371691, |
| 5.033994633121257e-44 |
| ], |
| "nsamples": 209 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7140924962133864, |
| "mean": 0.7236861408669109, |
| "wmean": 0.721280701233954 |
| }, |
| "spearman": { |
| "all": 0.7490972681198437, |
| "mean": 0.7564266193418148, |
| "wmean": 0.7548021651740933 |
| } |
| } |
| }, |
| "eval_senteval-STS12": 0.6520942930121344, |
| "eval_senteval-STS13": 0.7489808796663362, |
| "eval_senteval-STS14": 0.6214541243526541, |
| "eval_senteval-STS15": 0.7497940183329284, |
| "eval_senteval-STS16": 0.7490972681198437, |
| "eval_senteval-STSBenchmark": 0.708944371429387, |
| "eval_senteval-SICKRelatedness": 0.7122486034563766, |
| "eval_senteval-avg_sts_7": 0.7060876511956659, |
| "eval_senteval-MR": 80.75, |
| "eval_senteval-CR": 87.02, |
| "eval_senteval-SUBJ": 95.02, |
| "eval_senteval-MPQA": 88.67, |
| "eval_senteval-SST2": 82.8, |
| "eval_senteval-TREC": 78.63, |
| "eval_senteval-MRPC": 70.17, |
| "eval_senteval-avg_transfer": 83.2942857142857 |
| } |
|
|
