| { |
| "num_threads": 224, |
| "split_by_whitespace": true, |
| "model_type": "unigram", |
| "vocab_size": 250680, |
| "character_coverage": 0.9999, |
| "byte_fallback": true, |
| "split_by_number": true, |
| "split_digits": true, |
| "normalization_rule_name": "nfkc", |
| "max_sentence_length": 4096, |
| "shuffle_input_sentence": true, |
| "input_sentence_size": 0, |
| "train_extremely_large_corpus": true, |
| "allow_whitespace_only_pieces": true, |
| "required_chars": "", |
| "remove_extra_whitespaces": false, |
| "user_defined_symbols": [ |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<eod>", |
| "<placeholder_tok_0>", |
| "<placeholder_tok_1>", |
| "<placeholder_tok_2>", |
| "<placeholder_tok_3>", |
| "<placeholder_tok_4>", |
| "<placeholder_tok_5>", |
| "<placeholder_tok_6>", |
| "<placeholder_tok_7>", |
| "<placeholder_tok_8>", |
| "<placeholder_tok_9>", |
| "<placeholder_tok_10>", |
| "<placeholder_tok_11>", |
| "<placeholder_tok_12>", |
| "<placeholder_tok_13>", |
| "<placeholder_tok_14>", |
| "<placeholder_tok_15>", |
| "<placeholder_tok_16>", |
| "<placeholder_tok_17>", |
| "<placeholder_tok_18>", |
| "<placeholder_tok_19>", |
| "<placeholder_tok_20>", |
| "<placeholder_tok_21>", |
| "<placeholder_tok_22>", |
| "<placeholder_tok_23>", |
| "<placeholder_tok_24>", |
| "<placeholder_tok_25>", |
| "<placeholder_tok_26>", |
| "<placeholder_tok_27>", |
| "<placeholder_tok_28>", |
| "<placeholder_tok_29>", |
| "<placeholder_tok_30>", |
| "<placeholder_tok_31>", |
| "<placeholder_tok_32>", |
| "<placeholder_tok_33>", |
| "<placeholder_tok_34>", |
| "<placeholder_tok_35>", |
| "<placeholder_tok_36>", |
| "<placeholder_tok_37>", |
| "<placeholder_tok_38>", |
| "<placeholder_tok_39>", |
| "<placeholder_tok_40>", |
| "<placeholder_tok_41>", |
| "<placeholder_tok_42>", |
| "<placeholder_tok_43>", |
| "<placeholder_tok_44>", |
| "<placeholder_tok_45>", |
| "<placeholder_tok_46>", |
| "<placeholder_tok_47>", |
| "<placeholder_tok_48>", |
| "<placeholder_tok_49>", |
| "<placeholder_tok_50>", |
| "<placeholder_tok_51>", |
| "<placeholder_tok_52>", |
| "<placeholder_tok_53>", |
| "<placeholder_tok_54>", |
| "<placeholder_tok_55>", |
| "<placeholder_tok_56>", |
| "<placeholder_tok_57>", |
| "<placeholder_tok_58>", |
| "<placeholder_tok_59>", |
| "<placeholder_tok_60>", |
| "<placeholder_tok_61>", |
| "<placeholder_tok_62>", |
| "<placeholder_tok_63>", |
| "<placeholder_tok_64>", |
| "<placeholder_tok_65>", |
| "<placeholder_tok_66>", |
| "<placeholder_tok_67>", |
| "<placeholder_tok_68>", |
| "<placeholder_tok_69>", |
| "<placeholder_tok_70>", |
| "<placeholder_tok_71>", |
| "<placeholder_tok_72>", |
| "<placeholder_tok_73>", |
| "<placeholder_tok_74>", |
| "<placeholder_tok_75>", |
| "<placeholder_tok_76>", |
| "<placeholder_tok_77>", |
| "<placeholder_tok_78>", |
| "<placeholder_tok_79>", |
| "<placeholder_tok_80>", |
| "<placeholder_tok_81>", |
| "<placeholder_tok_82>", |
| "<placeholder_tok_83>", |
| "<placeholder_tok_84>", |
| "<placeholder_tok_85>", |
| "<placeholder_tok_86>", |
| "<placeholder_tok_87>", |
| "<placeholder_tok_88>", |
| "<placeholder_tok_89>", |
| "<placeholder_tok_90>", |
| "<placeholder_tok_91>", |
| "<placeholder_tok_92>", |
| "<placeholder_tok_93>", |
| "<placeholder_tok_94>", |
| "<placeholder_tok_95>", |
| "<placeholder_tok_96>", |
| "<placeholder_tok_97>", |
| "<placeholder_tok_98>", |
| "<placeholder_tok_99>", |
| "<placeholder_tok_100>", |
| "<placeholder_tok_101>", |
| "<placeholder_tok_102>", |
| "<placeholder_tok_103>", |
| "<placeholder_tok_104>", |
| "<placeholder_tok_105>", |
| "<placeholder_tok_106>", |
| "<placeholder_tok_107>", |
| "<placeholder_tok_108>", |
| "<placeholder_tok_109>", |
| "<placeholder_tok_110>", |
| "<placeholder_tok_111>", |
| "<placeholder_tok_112>", |
| "<placeholder_tok_113>", |
| "<placeholder_tok_114>", |
| "<placeholder_tok_115>", |
| "<placeholder_tok_116>", |
| "<placeholder_tok_117>", |
| "<placeholder_tok_118>", |
| "<placeholder_tok_119>", |
| "<placeholder_tok_120>", |
| "<placeholder_tok_121>", |
| "<placeholder_tok_122>", |
| "<placeholder_tok_123>", |
| "<placeholder_tok_124>", |
| "<placeholder_tok_125>", |
| "<placeholder_tok_126>", |
| "<placeholder_tok_127>", |
| "<placeholder_tok_128>", |
| "<placeholder_tok_129>", |
| "<placeholder_tok_130>", |
| "<placeholder_tok_131>", |
| "<placeholder_tok_132>", |
| "<placeholder_tok_133>", |
| "<placeholder_tok_134>", |
| "<placeholder_tok_135>", |
| "<placeholder_tok_136>", |
| "<placeholder_tok_137>", |
| "<placeholder_tok_138>", |
| "<placeholder_tok_139>", |
| "<placeholder_tok_140>", |
| "<placeholder_tok_141>", |
| "<placeholder_tok_142>", |
| "<placeholder_tok_143>", |
| "<placeholder_tok_144>", |
| "<placeholder_tok_145>", |
| "<placeholder_tok_146>", |
| "<placeholder_tok_147>", |
| "<placeholder_tok_148>", |
| "<placeholder_tok_149>", |
| "<placeholder_tok_150>", |
| "<placeholder_tok_151>", |
| "<placeholder_tok_152>", |
| "<placeholder_tok_153>", |
| "<placeholder_tok_154>", |
| "<placeholder_tok_155>", |
| "<placeholder_tok_156>", |
| "<placeholder_tok_157>", |
| "<placeholder_tok_158>", |
| "<placeholder_tok_159>", |
| "<placeholder_tok_160>", |
| "<placeholder_tok_161>", |
| "<placeholder_tok_162>", |
| "<placeholder_tok_163>", |
| "<placeholder_tok_164>", |
| "<placeholder_tok_165>", |
| "<placeholder_tok_166>", |
| "<placeholder_tok_167>", |
| "<placeholder_tok_168>", |
| "<placeholder_tok_169>", |
| "<placeholder_tok_170>", |
| "<placeholder_tok_171>", |
| "<placeholder_tok_172>", |
| "<placeholder_tok_173>", |
| "<placeholder_tok_174>", |
| "<placeholder_tok_175>", |
| "<placeholder_tok_176>", |
| "<placeholder_tok_177>", |
| "<placeholder_tok_178>", |
| "<placeholder_tok_179>", |
| "<placeholder_tok_180>", |
| "<placeholder_tok_181>", |
| "<placeholder_tok_182>", |
| "<placeholder_tok_183>", |
| "<placeholder_tok_184>", |
| "<placeholder_tok_185>", |
| "<placeholder_tok_186>", |
| "<placeholder_tok_187>", |
| "<placeholder_tok_188>", |
| "<placeholder_tok_189>", |
| "<placeholder_tok_190>", |
| "<placeholder_tok_191>", |
| "<placeholder_tok_192>", |
| "<placeholder_tok_193>", |
| "<placeholder_tok_194>", |
| "<placeholder_tok_195>", |
| "<placeholder_tok_196>", |
| "<placeholder_tok_197>", |
| "<placeholder_tok_198>", |
| "<placeholder_tok_199>", |
| "<placeholder_tok_200>", |
| "<placeholder_tok_201>", |
| "<placeholder_tok_202>", |
| "<placeholder_tok_203>", |
| "<placeholder_tok_204>", |
| "<placeholder_tok_205>", |
| "<placeholder_tok_206>", |
| "<placeholder_tok_207>", |
| "<placeholder_tok_208>", |
| "<placeholder_tok_209>", |
| "<placeholder_tok_210>", |
| "<placeholder_tok_211>", |
| "<placeholder_tok_212>", |
| "<placeholder_tok_213>", |
| "<placeholder_tok_214>", |
| "<placeholder_tok_215>", |
| "<placeholder_tok_216>", |
| "<placeholder_tok_217>", |
| "<placeholder_tok_218>", |
| "<placeholder_tok_219>", |
| "<placeholder_tok_220>", |
| "<placeholder_tok_221>", |
| "<placeholder_tok_222>", |
| "<placeholder_tok_223>", |
| "<placeholder_tok_224>", |
| "<placeholder_tok_225>", |
| "<placeholder_tok_226>", |
| "<placeholder_tok_227>", |
| "<placeholder_tok_228>", |
| "<placeholder_tok_229>", |
| "<placeholder_tok_230>", |
| "<placeholder_tok_231>", |
| "<placeholder_tok_232>", |
| "<placeholder_tok_233>", |
| "<placeholder_tok_234>", |
| "<placeholder_tok_235>", |
| "<placeholder_tok_236>", |
| "<placeholder_tok_237>", |
| "<placeholder_tok_238>", |
| "<placeholder_tok_239>", |
| "<placeholder_tok_240>", |
| "<placeholder_tok_241>", |
| "<placeholder_tok_242>", |
| "<placeholder_tok_243>", |
| "<placeholder_tok_244>", |
| "<placeholder_tok_245>", |
| "<placeholder_tok_246>", |
| "<placeholder_tok_247>", |
| "<placeholder_tok_248>", |
| "<placeholder_tok_249>", |
| "<placeholder_tok_250>", |
| "<placeholder_tok_251>", |
| "<placeholder_tok_252>", |
| "<placeholder_tok_253>", |
| "<placeholder_tok_254>", |
| "<placeholder_tok_255>" |
| ], |
| "datasets_dir": "/home/fhgiais/gptx_ablations/bias_analysis/data/tokenizer/temp/", |
| "save_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24", |
| "text_key": "text", |
| "cache_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24/cache", |
| "library": "sentencepiece", |
| "auto_map": { |
| "AutoTokenizer": [ |
| "gptx_tokenizer.SPTokenizer", |
| null |
| ] |
| }, |
| "tokenizer_class": "SPTokenizer" |
| } |
