Spaces:

OliverPerrin
/

LexiMind

Sleeping

OliverPerrin commited on Dec 5, 2025

Commit

97abce2

1 Parent(s): 40ccedf

Fix: Regenerate tokenizer without add_prefix_space to fix CI

Files changed (1) hide show

artifacts/hf_tokenizer/tokenizer_config.json CHANGED Viewed

@@ -1,5 +1,5 @@
 {
-  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
@@ -932,7 +932,6 @@
   "eos_token": "</s>",
   "extra_ids": 100,
   "extra_special_tokens": {},
-  "legacy": true,
   "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},

 {
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
   "eos_token": "</s>",
   "extra_ids": 100,
   "extra_special_tokens": {},
   "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},