| { |
| "added_tokens_decoder": { |
| "1": { |
| "content": "<s>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": true |
| }, |
| "2": { |
| "content": "</s>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": true |
| }, |
| "3": { |
| "content": "<pad>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": true |
| }, |
| "4": { |
| "content": "<|im_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "5": { |
| "content": "<|im_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "6": { |
| "content": "<|im_sp_00|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "7": { |
| "content": "<|im_sp_01|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "8": { |
| "content": "<|im_sp_02|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "9": { |
| "content": "<|im_sp_03|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "10": { |
| "content": "<|im_sp_04|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "11": { |
| "content": "<|im_sp_05|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "12": { |
| "content": "<|im_sp_06|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "13": { |
| "content": "<|im_sp_07|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "14": { |
| "content": "<|im_sp_08|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "15": { |
| "content": "<|im_sp_09|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "16": { |
| "content": "<|im_sp_10|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "17": { |
| "content": "<|im_sp_11|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "18": { |
| "content": "<|im_sp_12|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "19": { |
| "content": "<|im_sp_13|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "20": { |
| "content": "<|im_sp_14|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "21": { |
| "content": "<|im_sp_15|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "22": { |
| "content": "<|im_sp_16|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "23": { |
| "content": "<|im_sp_17|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "24": { |
| "content": "<|im_sp_18|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25": { |
| "content": "<|im_sp_19|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "26": { |
| "content": "<|im_sp_20|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "27": { |
| "content": "<|im_sp_21|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "28": { |
| "content": "<|im_sp_22|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "29": { |
| "content": "<|im_sp_23|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "30": { |
| "content": "<|im_sp_24|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "31": { |
| "content": "<|im_sp_25|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "32": { |
| "content": "<|im_sp_26|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "33": { |
| "content": "<|im_sp_27|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "34": { |
| "content": "<|im_sp_28|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "35": { |
| "content": "<|im_sp_29|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "36": { |
| "content": "<|im_sp_30|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "37": { |
| "content": "<|im_sp_31|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "38": { |
| "content": "<|im_sp_32|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "39": { |
| "content": "<|im_sp_33|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "40": { |
| "content": "<|im_sp_34|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "41": { |
| "content": "<|im_sp_35|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "42": { |
| "content": "<|im_sp_36|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "43": { |
| "content": "<|im_sp_37|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "44": { |
| "content": "<|im_sp_38|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "45": { |
| "content": "<|im_sp_39|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "46": { |
| "content": "<|im_sp_40|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "47": { |
| "content": "<|im_sp_41|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "48": { |
| "content": "<|im_sp_42|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "49": { |
| "content": "<|im_sp_43|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "50": { |
| "content": "<|im_sp_44|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "51": { |
| "content": "<|im_sp_45|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "52": { |
| "content": "<|im_sp_46|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "53": { |
| "content": "<|im_sp_47|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "54": { |
| "content": "<|im_sp_48|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "55": { |
| "content": "<|im_sp_49|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "56": { |
| "content": "<|im_sp_50|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "57": { |
| "content": "<|im_sp_51|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "58": { |
| "content": "<|im_sp_52|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "59": { |
| "content": "<|im_sp_53|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "60": { |
| "content": "<|im_sp_54|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "61": { |
| "content": "<|im_sp_55|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "62": { |
| "content": "<|im_sp_56|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "63": { |
| "content": "<|im_sp_57|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "64": { |
| "content": "<|im_sp_58|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "65": { |
| "content": "<|im_sp_59|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "66": { |
| "content": "<|im_sp_60|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "67": { |
| "content": "<|im_sp_61|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "68": { |
| "content": "<|im_sp_62|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "69": { |
| "content": "<|im_sp_63|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "70": { |
| "content": "<|im_sp_64|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "71": { |
| "content": "<|im_sp_65|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "72": { |
| "content": "<|im_sp_66|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "73": { |
| "content": "<|im_sp_67|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "74": { |
| "content": "<|im_sp_68|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "75": { |
| "content": "<|im_sp_69|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "76": { |
| "content": "<|im_sp_70|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "77": { |
| "content": "<|im_sp_71|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "78": { |
| "content": "<|im_sp_72|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "79": { |
| "content": "<|im_sp_73|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "80": { |
| "content": "<|im_sp_74|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "81": { |
| "content": "<|im_sp_75|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "82": { |
| "content": "<|im_sp_76|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "83": { |
| "content": "<|im_sp_77|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "84": { |
| "content": "<|im_sp_78|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "85": { |
| "content": "<|im_sp_79|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "86": { |
| "content": "<|im_sp_80|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "87": { |
| "content": "<|im_sp_81|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "88": { |
| "content": "<|im_sp_82|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "89": { |
| "content": "<|im_sp_83|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "90": { |
| "content": "<|im_sp_84|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "91": { |
| "content": "<|im_sp_85|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "92": { |
| "content": "<|im_sp_86|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "93": { |
| "content": "<|im_sp_87|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "94": { |
| "content": "<|im_sp_88|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "95": { |
| "content": "<|im_sp_89|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "96": { |
| "content": "<|im_sp_90|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "97": { |
| "content": "<|im_sp_91|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "98": { |
| "content": "<|im_sp_92|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "99": { |
| "content": "<|im_sp_93|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "100": { |
| "content": "<|im_sp_94|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "101": { |
| "content": "<|im_sp_95|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "102": { |
| "content": "<|im_sp_96|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "103": { |
| "content": "<|im_sp_97|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "104": { |
| "content": "<|im_sp_98|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "105": { |
| "content": "<|im_sp_99|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "116": { |
| "content": "\n", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| } |
| }, |
| "clean_up_tokenization_spaces": false, |
| "extra_special_tokens": {}, |
| "max_length": null, |
| "model_input_names": [ |
| "input_ids", |
| "attention_mask" |
| ], |
| "model_max_length": 1000000000000000019884624838656, |
| "pad_to_multiple_of": null, |
| "pad_token": "<pad>", |
| "pad_token_type_id": 0, |
| "padding_side": "right", |
| "tokenizer_class": "PreTrainedTokenizerFast" |
| } |
|
|
