| { |
| "STSBenchmark": { |
| "train": { |
| "pearson": [ |
| 0.7675016313639117, |
| 0.0 |
| ], |
| "spearman": [ |
| 0.7404488937353225, |
| 0.0 |
| ], |
| "nsamples": 5749 |
| }, |
| "dev": { |
| "pearson": [ |
| 0.7658521041744508, |
| 1.4020329449305176e-289 |
| ], |
| "spearman": [ |
| 0.7730453513050343, |
| 2.066561283121993e-298 |
| ], |
| "nsamples": 1500 |
| }, |
| "test": { |
| "pearson": [ |
| 0.7623056519318484, |
| 1.8448021245198884e-262 |
| ], |
| "spearman": [ |
| 0.7572290027800557, |
| 5.3120116722114526e-257 |
| ], |
| "nsamples": 1379 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7647570029076096, |
| "mean": 0.7652197958234037, |
| "wmean": 0.7663843913985655 |
| }, |
| "spearman": { |
| "all": 0.7534738635333297, |
| "mean": 0.7569077492734708, |
| "wmean": 0.7487978108339844 |
| } |
| } |
| }, |
| "SICKRelatedness": { |
| "train": { |
| "pearson": [ |
| 0.7707517065205873, |
| 0.0 |
| ], |
| "spearman": [ |
| 0.6949522587450165, |
| 0.0 |
| ], |
| "nsamples": 4500 |
| }, |
| "dev": { |
| "pearson": [ |
| 0.778349180191507, |
| 9.711324138536703e-103 |
| ], |
| "spearman": [ |
| 0.7176593438481393, |
| 2.743332620783351e-80 |
| ], |
| "nsamples": 500 |
| }, |
| "test": { |
| "pearson": [ |
| 0.7642987657770804, |
| 0.0 |
| ], |
| "spearman": [ |
| 0.6847455515560564, |
| 0.0 |
| ], |
| "nsamples": 4927 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7678373187987577, |
| "mean": 0.7711332174963915, |
| "wmean": 0.7679316297393041 |
| }, |
| "spearman": { |
| "all": 0.6909477744903625, |
| "mean": 0.6991190513830707, |
| "wmean": 0.6910301368785469 |
| } |
| } |
| }, |
| "MR": { |
| "devacc": 81.19, |
| "acc": 80.75, |
| "ndev": 10662, |
| "ntest": 10662 |
| }, |
| "CR": { |
| "devacc": 87.12, |
| "acc": 86.73, |
| "ndev": 3775, |
| "ntest": 3775 |
| }, |
| "SUBJ": { |
| "devacc": 95.61, |
| "acc": 95.29, |
| "ndev": 10000, |
| "ntest": 10000 |
| }, |
| "MPQA": { |
| "devacc": 89.04, |
| "acc": 88.61, |
| "ndev": 10606, |
| "ntest": 10606 |
| }, |
| "SST2": { |
| "devacc": 85.21, |
| "acc": 85.61, |
| "ndev": 872, |
| "ntest": 1821 |
| }, |
| "TREC": { |
| "devacc": 80.69, |
| "acc": 86.8, |
| "ndev": 5452, |
| "ntest": 500 |
| }, |
| "MRPC": { |
| "devacc": 71.69, |
| "acc": 70.32, |
| "f1": 79.57, |
| "ndev": 4076, |
| "ntest": 1725 |
| }, |
| "STS12": { |
| "MSRpar": { |
| "pearson": [ |
| 0.4514113169306324, |
| 6.251817668045887e-39 |
| ], |
| "spearman": [ |
| 0.4499549654577979, |
| 1.1613312893486279e-38 |
| ], |
| "nsamples": 750 |
| }, |
| "MSRvid": { |
| "pearson": [ |
| 0.8599559900295872, |
| 1.0643859926359198e-220 |
| ], |
| "spearman": [ |
| 0.8649974875962446, |
| 3.229629761923383e-226 |
| ], |
| "nsamples": 750 |
| }, |
| "SMTeuroparl": { |
| "pearson": [ |
| 0.4931333562119396, |
| 1.6812261260529903e-29 |
| ], |
| "spearman": [ |
| 0.6128103790153274, |
| 1.1381831791276455e-48 |
| ], |
| "nsamples": 459 |
| }, |
| "surprise.OnWN": { |
| "pearson": [ |
| 0.7365214189654901, |
| 3.947242882144461e-129 |
| ], |
| "spearman": [ |
| 0.683719802480883, |
| 1.910512439440496e-104 |
| ], |
| "nsamples": 750 |
| }, |
| "surprise.SMTnews": { |
| "pearson": [ |
| 0.6084967943558258, |
| 8.882847362372895e-42 |
| ], |
| "spearman": [ |
| 0.6324009475046439, |
| 5.9238371155257115e-46 |
| ], |
| "nsamples": 399 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.6090786981125251, |
| "mean": 0.629903775298695, |
| "wmean": 0.6451270836208292 |
| }, |
| "spearman": { |
| "all": 0.5915463354166586, |
| "mean": 0.6487767164109794, |
| "wmean": 0.6539936080030831 |
| } |
| } |
| }, |
| "STS13": { |
| "FNWN": { |
| "pearson": [ |
| 0.5827724320641434, |
| 1.399853624093483e-18 |
| ], |
| "spearman": [ |
| 0.6243224304995846, |
| 8.1205413287147755e-22 |
| ], |
| "nsamples": 189 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.782141211640678, |
| 7.962032809197313e-156 |
| ], |
| "spearman": [ |
| 0.7864730688106963, |
| 1.0734436155388987e-158 |
| ], |
| "nsamples": 750 |
| }, |
| "OnWN": { |
| "pearson": [ |
| 0.7763285115488442, |
| 3.939259823816713e-114 |
| ], |
| "spearman": [ |
| 0.7706743357210359, |
| 1.7455185991511694e-111 |
| ], |
| "nsamples": 561 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7269930269261002, |
| "mean": 0.7137473850845552, |
| "wmean": 0.7548467955796887 |
| }, |
| "spearman": { |
| "all": 0.7392805868780532, |
| "mean": 0.7271566116771057, |
| "wmean": 0.7601333622079632 |
| } |
| } |
| }, |
| "STS14": { |
| "deft-forum": { |
| "pearson": [ |
| 0.5228897751421426, |
| 6.073682402382636e-33 |
| ], |
| "spearman": [ |
| 0.5053330319071232, |
| 1.532862819818703e-30 |
| ], |
| "nsamples": 450 |
| }, |
| "deft-news": { |
| "pearson": [ |
| 0.7680069983657648, |
| 1.2852042740473546e-59 |
| ], |
| "spearman": [ |
| 0.7189884605027314, |
| 5.271976030712246e-49 |
| ], |
| "nsamples": 300 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.7541282525177266, |
| 9.793324818317855e-139 |
| ], |
| "spearman": [ |
| 0.7184718603499471, |
| 4.702376618230486e-120 |
| ], |
| "nsamples": 750 |
| }, |
| "images": { |
| "pearson": [ |
| 0.8478349220740108, |
| 2.8453760358482384e-208 |
| ], |
| "spearman": [ |
| 0.8081484798616755, |
| 3.9047867377054607e-174 |
| ], |
| "nsamples": 750 |
| }, |
| "OnWN": { |
| "pearson": [ |
| 0.8215837025605404, |
| 9.871899662708195e-185 |
| ], |
| "spearman": [ |
| 0.8279072975227956, |
| 4.92735009919912e-190 |
| ], |
| "nsamples": 750 |
| }, |
| "tweet-news": { |
| "pearson": [ |
| 0.7317126218269044, |
| 1.2202543536716308e-126 |
| ], |
| "spearman": [ |
| 0.6520317484632859, |
| 5.3026345548268765e-92 |
| ], |
| "nsamples": 750 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7189696594882792, |
| "mean": 0.7410260454145149, |
| "wmean": 0.7552392326821546 |
| }, |
| "spearman": { |
| "all": 0.670800547509326, |
| "mean": 0.7051468131012597, |
| "wmean": 0.7194709179086142 |
| } |
| } |
| }, |
| "STS15": { |
| "answers-forums": { |
| "pearson": [ |
| 0.7262407045308339, |
| 1.1025031075653589e-62 |
| ], |
| "spearman": [ |
| 0.7275038094083748, |
| 5.324790051370908e-63 |
| ], |
| "nsamples": 375 |
| }, |
| "answers-students": { |
| "pearson": [ |
| 0.6978521677757564, |
| 1.6036590447498983e-110 |
| ], |
| "spearman": [ |
| 0.7024365108998977, |
| 1.4353461268590496e-112 |
| ], |
| "nsamples": 750 |
| }, |
| "belief": { |
| "pearson": [ |
| 0.7839138106441258, |
| 3.235687910099277e-79 |
| ], |
| "spearman": [ |
| 0.79336254174086, |
| 2.047534842473332e-82 |
| ], |
| "nsamples": 375 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.8124656869189333, |
| 1.9047661503782033e-177 |
| ], |
| "spearman": [ |
| 0.8124179763234277, |
| 2.0744785143960583e-177 |
| ], |
| "nsamples": 750 |
| }, |
| "images": { |
| "pearson": [ |
| 0.8731830765774482, |
| 1.1398312421407709e-235 |
| ], |
| "spearman": [ |
| 0.8795017881810903, |
| 1.9914488519259588e-243 |
| ], |
| "nsamples": 750 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7827019127153672, |
| "mean": 0.7787310892894196, |
| "wmean": 0.7846445472149044 |
| }, |
| "spearman": { |
| "all": 0.790296617367925, |
| "mean": 0.7830445253107301, |
| "wmean": 0.7886973627447582 |
| } |
| } |
| }, |
| "STS16": { |
| "answer-answer": { |
| "pearson": [ |
| 0.7325081931498698, |
| 5.607126931548686e-44 |
| ], |
| "spearman": [ |
| 0.7313808706110454, |
| 8.788498983014453e-44 |
| ], |
| "nsamples": 254 |
| }, |
| "headlines": { |
| "pearson": [ |
| 0.7702918518354713, |
| 3.5986232616752304e-50 |
| ], |
| "spearman": [ |
| 0.7838189633800285, |
| 5.030858666324319e-53 |
| ], |
| "nsamples": 249 |
| }, |
| "plagiarism": { |
| "pearson": [ |
| 0.8193777086614761, |
| 5.14320199251657e-57 |
| ], |
| "spearman": [ |
| 0.8274487763356487, |
| 4.601518050831221e-59 |
| ], |
| "nsamples": 230 |
| }, |
| "postediting": { |
| "pearson": [ |
| 0.8373109699109689, |
| 2.1202879642927884e-65 |
| ], |
| "spearman": [ |
| 0.8554383865248261, |
| 4.218785247850647e-71 |
| ], |
| "nsamples": 244 |
| }, |
| "question-question": { |
| "pearson": [ |
| 0.7703959894794563, |
| 2.4658565942110764e-42 |
| ], |
| "spearman": [ |
| 0.7772827176027448, |
| 1.5625377662687375e-43 |
| ], |
| "nsamples": 209 |
| }, |
| "all": { |
| "pearson": { |
| "all": 0.7784793728977035, |
| "mean": 0.7859769426074485, |
| "wmean": 0.785525517385094 |
| }, |
| "spearman": { |
| "all": 0.7906179855205325, |
| "mean": 0.7950739428908588, |
| "wmean": 0.7946323236636283 |
| } |
| } |
| }, |
| "eval_senteval-STS12": 0.5915463354166586, |
| "eval_senteval-STS13": 0.7392805868780532, |
| "eval_senteval-STS14": 0.670800547509326, |
| "eval_senteval-STS15": 0.790296617367925, |
| "eval_senteval-STS16": 0.7906179855205325, |
| "eval_senteval-STSBenchmark": 0.7534738635333297, |
| "eval_senteval-SICKRelatedness": 0.6909477744903625, |
| "eval_senteval-avg_sts_7": 0.7181376729594552, |
| "eval_senteval-MR": 81.19, |
| "eval_senteval-CR": 87.12, |
| "eval_senteval-SUBJ": 95.61, |
| "eval_senteval-MPQA": 89.04, |
| "eval_senteval-SST2": 85.21, |
| "eval_senteval-TREC": 80.69, |
| "eval_senteval-MRPC": 71.69, |
| "eval_senteval-avg_transfer": 84.36428571428571 |
| } |
|
|
