Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,5 @@
+# Recovered LoRA Adapter
+
+- Base model: `jeongseokoh/llama3.1_8b_sft-llopa-k24-no_system`
+- Source checkpoint: `/data3/LLoPA/Capsule/model_from_downstream/llama3.1_8b_sft-llopa-k24-no_system-opencode-train.code.q60000-llopa-k24-no_system/epoch_0`
+- Recovery source: `Capsule/recover_accelerate_lora_checkpoint.py`

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "jeongseokoh/llama3.1_8b_sft-llopa-k24-no_system",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64.0,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "v_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3905004ba399131324d33dc934caafb759658b674660250a3b7d757202a2c698
+size 2437082304

backbone.json

@@ -0,0 +1,3 @@
+{
+  "backbone": "jeongseokoh/llama3.1_8b_sft-llopa-k24-no_system"
+}

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,46 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModelForCausalLM": "modeling_llopa.LLOPAForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "capsule_llopa_enabled": true,
+  "capsule_lower_layers": 24,
+  "capsule_no_upper_attn": false,
+  "capsule_prefill_attn": "causal",
+  "capsule_prefill_mode": "lower",
+  "capsule_system_prefill": "no_system",
+  "capsule_user_prefill": "full",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "llopa_num_specials": 0,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": true,
+  "vocab_size": 128256
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.55.4"
+}

llopa_utils/saving_utils.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+import torch
+
+
+EMBEDDING_FILENAME = "embeddings.pt"
+BACKBONE_REF_FILENAME = "backbone.json"
+LLOPA_SPECIALS_FILENAME = "llopa_specials.pt"
+
+
+def write_backbone_ref(best_dir: str | Path, backbone: str | None) -> None:
+    if not backbone:
+        return
+    out = Path(best_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    payload = {"backbone": str(backbone)}
+    (out / BACKBONE_REF_FILENAME).write_text(
+        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+
+
+def read_backbone_ref(best_dir: str | Path) -> Optional[str]:
+    path = Path(best_dir) / BACKBONE_REF_FILENAME
+    if not path.is_file():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    val = data.get("backbone")
+    return str(val) if val else None
+
+
+def save_embedding_layer(model, best_dir: str | Path, *, include_lm_head: bool = True) -> bool:
+    out = Path(best_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    try:
+        emb = model.get_input_embeddings()
+    except Exception:
+        emb = None
+    if emb is None or not hasattr(emb, "weight"):
+        return False
+    payload: dict[str, torch.Tensor] = {
+        "input_embeddings": emb.weight.detach().cpu()
+    }
+    if include_lm_head:
+        head = getattr(model, "lm_head", None)
+        if head is None and hasattr(model, "get_output_embeddings"):
+            try:
+                head = model.get_output_embeddings()
+            except Exception:
+                head = None
+        if head is not None and hasattr(head, "weight"):
+            try:
+                same_storage = head.weight.data_ptr() == emb.weight.data_ptr()
+            except Exception:
+                same_storage = False
+            if not same_storage and head.weight.shape == emb.weight.shape:
+                payload["lm_head"] = head.weight.detach().cpu()
+    torch.save(payload, out / EMBEDDING_FILENAME)
+    return True
+
+
+def load_embedding_layer(model, best_dir: str | Path) -> bool:
+    path = Path(best_dir) / EMBEDDING_FILENAME
+    if not path.is_file():
+        return False
+    try:
+        payload = torch.load(path, map_location="cpu")
+    except Exception:
+        return False
+    weight = payload.get("input_embeddings")
+    if weight is None:
+        return False
+    try:
+        cur_emb = model.get_input_embeddings()
+    except Exception:
+        cur_emb = None
+    if cur_emb is None or not hasattr(cur_emb, "weight"):
+        return False
+    if cur_emb.weight.shape[0] != weight.shape[0]:
+        try:
+            model.resize_token_embeddings(weight.shape[0])
+        except Exception:
+            pass
+        cur_emb = model.get_input_embeddings()
+    cur_emb.weight.data.copy_(weight.to(cur_emb.weight.dtype))
+    head_weight = payload.get("lm_head")
+    if head_weight is not None:
+        head = getattr(model, "lm_head", None)
+        if head is None and hasattr(model, "get_output_embeddings"):
+            try:
+                head = model.get_output_embeddings()
+            except Exception:
+                head = None
+        if head is not None and hasattr(head, "weight") and head.weight.shape == head_weight.shape:
+            head.weight.data.copy_(head_weight.to(head.weight.dtype))
+    return True
+
+
+def _get_llopa_specials_module(model):
+    for cand in (model, getattr(model, "base_model", None), getattr(model, "model", None), getattr(model, "transformer", None)):
+        if cand is not None and hasattr(cand, "llopa_specials"):
+            return cand
+    return None
+
+
+def save_llopa_specials(model, best_dir: str | Path) -> bool:
+    out = Path(best_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    core = _get_llopa_specials_module(model)
+    if core is None:
+        return False
+    specials = getattr(core, "llopa_specials", None)
+    if specials is None:
+        return False
+    try:
+        tensors = [p.detach().cpu() for p in specials]
+    except Exception:
+        return False
+    if not tensors:
+        return False
+    payload = {
+        "llopa_num_specials": int(getattr(core, "llopa_num_specials", 0) or 0),
+        "llopa_num_layers": len(tensors),
+        "tensors": tensors,
+    }
+    torch.save(payload, out / LLOPA_SPECIALS_FILENAME)
+    return True
+
+
+def load_llopa_specials(model, best_dir: str | Path) -> bool:
+    path = Path(best_dir) / LLOPA_SPECIALS_FILENAME
+    if not path.is_file():
+        return False
+    try:
+        payload = torch.load(path, map_location="cpu")
+    except Exception:
+        return False
+    tensors = payload.get("tensors")
+    if not tensors:
+        return False
+    core = _get_llopa_specials_module(model)
+    if core is None:
+        return False
+    specials = getattr(core, "llopa_specials", None)
+    if specials is None:
+        return False
+    if len(specials) != len(tensors):
+        return False
+    try:
+        for dst, src in zip(specials, tensors):
+            if dst.shape != src.shape:
+                return False
+            dst.data.copy_(src.to(dst.dtype))
+    except Exception:
+        return False
+    return True
+
+
+def init_llopa_specials_with_mean(model, *, chunk_size: int = 8192) -> bool:
+    core = _get_llopa_specials_module(model)
+    if core is None:
+        return False
+    specials = getattr(core, "llopa_specials", None)
+    if specials is None or len(specials) == 0:
+        return False
+
+    emb = None
+    try:
+        emb = model.get_input_embeddings()
+    except Exception:
+        emb = None
+    if emb is None and hasattr(core, "get_input_embeddings"):
+        try:
+            emb = core.get_input_embeddings()
+        except Exception:
+            emb = None
+    if emb is None:
+        emb = getattr(core, "embed_tokens", None)
+    if emb is None or not hasattr(emb, "weight"):
+        return False
+
+    weight = emb.weight.detach()
+    if weight.ndim != 2:
+        return False
+    old_num_tokens, hidden_size = weight.shape
+    if old_num_tokens <= 0:
+        return False
+
+    emb_cpu = weight.float().cpu()
+    mean = emb_cpu.mean(dim=0)
+    cov = torch.zeros((hidden_size, hidden_size), dtype=torch.float32)
+    step = max(1, int(chunk_size) if chunk_size else int(old_num_tokens))
+    for start in range(0, int(old_num_tokens), step):
+        chunk = emb_cpu[start : start + step]
+        centered = chunk - mean
+        cov += centered.t().matmul(centered)
+    cov /= float(old_num_tokens)
+    epsilon = 1e-9
+    cov_eps = cov * epsilon
+    try:
+        is_psd = torch.distributions.constraints.positive_definite.check(cov_eps).all()
+    except Exception:
+        is_psd = False
+
+    with torch.no_grad():
+        if bool(is_psd):
+            dist = torch.distributions.multivariate_normal.MultivariateNormal(
+                mean, covariance_matrix=cov_eps
+            )
+            for p in specials:
+                if p.numel() == 0:
+                    continue
+                samples = dist.sample(sample_shape=(p.shape[0],))
+                p.copy_(samples.to(dtype=p.dtype, device=p.device))
+        else:
+            mean_row = mean.unsqueeze(0)
+            for p in specials:
+                if p.numel() == 0:
+                    continue
+                p.copy_(mean_row.expand_as(p).to(dtype=p.dtype, device=p.device))
+    return True

llopa_utils/sniah_utils.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import re
+from typing import List, Tuple
+
+
+_QUESTION_RE = re.compile(
+    r"(What is the special magic .*?\\?)\\s*(?:The special magic .*?\\s+is\\s*)?$",
+    re.S,
+)
+_MEMORY_TAGS = ("<|memory_start|>", "<|memory_end|>")
+
+
+def extract_user_content(prompt: str) -> str:
+    """Extract the last user message content from a RULER-style prompt."""
+    if not isinstance(prompt, str):
+        return ""
+    start_token = "<|im_start|>user"
+    end_token = "<|im_end|>"
+    idx = prompt.rfind(start_token)
+    if idx != -1:
+        segment = prompt[idx + len(start_token):]
+        if segment.startswith("\n"):
+            segment = segment[1:]
+        end_idx = segment.find(end_token)
+        if end_idx != -1:
+            return segment[:end_idx].strip()
+    return prompt.strip()
+
+
+def _strip_memory_tags(text: str) -> str:
+    for tag in _MEMORY_TAGS:
+        text = text.replace(tag, "")
+    return text
+
+
+def split_sniah_user_content(content: str) -> Tuple[str, str]:
+    """Split RULER S-NIAH user content into (document, question)."""
+    if not isinstance(content, str):
+        return "", ""
+    content = _strip_memory_tags(content.strip())
+    if not content:
+        return "", ""
+
+    match = _QUESTION_RE.search(content)
+    if match:
+        question = match.group(1).strip()
+        doc = content[:match.start()].strip()
+        return doc, question
+
+    # Fallback: use the last question mark and the line it appears on.
+    q_idx = content.rfind("?")
+    if q_idx != -1:
+        line_start = content.rfind("\n", 0, q_idx)
+        if line_start == -1:
+            line_start = 0
+        question = content[line_start:q_idx + 1].strip()
+        doc = content[:line_start].strip()
+        return doc, question
+
+    return content, ""
+
+
+def extract_sniah_answers(extra_info: dict) -> List[str]:
+    answers = []
+    if not isinstance(extra_info, dict):
+        return answers
+    gt = extra_info.get("ground_truth")
+    if isinstance(gt, dict):
+        raw = gt.get("answers")
+        if isinstance(raw, list):
+            for item in raw:
+                if isinstance(item, str) and item.strip():
+                    answers.append(item.strip())
+        elif isinstance(raw, str) and raw.strip():
+            answers.append(raw.strip())
+    return answers

llopa_utils/structured_prompt.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+import contextlib
+
+import torch
+
+MISTRAL_ASSIST_START = "<Mistral_start>"
+
+
+def _is_mistral_template(tokenizer) -> bool:
+    tmpl = getattr(tokenizer, "chat_template", "") or ""
+    name = getattr(getattr(tokenizer, "init_kwargs", {}), "get", lambda k, d=None: d)(
+        "name_or_path", ""
+    )
+    return (
+        "[INST]" in tmpl
+        or "mistral" in str(name).lower()
+        or "mistral" in tmpl.lower()
+    )
+
+
+def _is_qwen3_tokenizer(tokenizer) -> bool:
+    name = getattr(tokenizer, "name_or_path", "") or ""
+    cls = tokenizer.__class__.__name__.lower()
+    return ("qwen3" in name.lower()) or ("qwen3" in cls)
+
+
+def apply_chat_template(tokenizer, messages, add_generation_prompt: bool):
+    force_thinking = getattr(tokenizer, "_force_enable_thinking", None)
+    if force_thinking is None and _is_qwen3_tokenizer(tokenizer):
+        force_thinking = False
+    try:
+        if force_thinking is None:
+            return tokenizer.apply_chat_template(
+                messages, tokenize=False, add_generation_prompt=add_generation_prompt
+            )
+        return tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=add_generation_prompt,
+            enable_thinking=bool(force_thinking),
+        )
+    except TypeError:
+        tmpl = getattr(tokenizer, "chat_template", "") or ""
+        try:
+            if force_thinking is None:
+                rendered = tokenizer.apply_chat_template(messages, tokenize=False)
+            else:
+                rendered = tokenizer.apply_chat_template(
+                    messages,
+                    tokenize=False,
+                    enable_thinking=bool(force_thinking),
+                )
+        except TypeError:
+            rendered = tokenizer.apply_chat_template(messages, tokenize=False)
+        if add_generation_prompt and "<|start_header_id|>" in tmpl:
+            rendered += "<|start_header_id|>assistant<|end_header_id|>\n\n"
+        return rendered
+
+
+def tokens_from_messages(tokenizer, messages, device, add_generation_prompt=False):
+    rendered = apply_chat_template(
+        tokenizer, messages, add_generation_prompt=add_generation_prompt
+    )
+    ids = tokenizer(
+        rendered, add_special_tokens=False, return_tensors="pt"
+    ).input_ids.to(device)
+    if add_generation_prompt and _is_mistral_template(tokenizer):
+        try:
+            tok_id = tokenizer.convert_tokens_to_ids(MISTRAL_ASSIST_START)
+            if tok_id is not None and tok_id != tokenizer.unk_token_id:
+                extra = torch.tensor([[int(tok_id)]], device=ids.device, dtype=ids.dtype)
+                ids = torch.cat([ids, extra], dim=1)
+        except Exception:
+            pass
+    return ids
+
+
+def lcp_len(a: torch.Tensor, b: torch.Tensor) -> int:
+    length = min(a.size(1), b.size(1))
+    eq = a[0, :length] == b[0, :length]
+    nz = (~eq).nonzero(as_tuple=False)
+    return int(nz[0, 0]) if nz.numel() else length
+
+
+def normalize_prompt_messages(messages):
+    out = []
+    if not isinstance(messages, list):
+        return out
+    for msg in messages:
+        if not isinstance(msg, dict):
+            continue
+        role = str(msg.get("role") or "user").strip().lower()
+        if role not in {"system", "user", "assistant"}:
+            role = "user"
+        content = str(msg.get("content") or "")
+        if role != "assistant":
+            content = content.strip()
+        if not content:
+            continue
+        out.append({"role": role, "content": content})
+    return out
+
+
+def _assistant_content_delta_from_messages(
+    tokenizer, prefix_messages, assistant_text: str, su_gen, device
+):
+    msgs_ass = list(prefix_messages) + [{"role": "assistant", "content": assistant_text}]
+    full_ids = tokens_from_messages(
+        tokenizer, msgs_ass, device, add_generation_prompt=False
+    )
+    if full_ids.size(1) <= su_gen.size(1):
+        return full_ids[:, :0]
+    return full_ids[:, su_gen.size(1) :]
+
+
+def _strip_trailing_assistant_stop_tokens(
+    tokenizer, token_ids: torch.Tensor
+) -> torch.Tensor:
+    if not isinstance(token_ids, torch.Tensor) or token_ids.numel() == 0:
+        return token_ids
+    stop_ids = set()
+    eos_id = getattr(tokenizer, "eos_token_id", None)
+    if eos_id is not None:
+        stop_ids.add(int(eos_id))
+    with contextlib.suppress(Exception):
+        eot_id = tokenizer.convert_tokens_to_ids("<|eot_id|>")
+        if eot_id is not None and eot_id != tokenizer.unk_token_id:
+            stop_ids.add(int(eot_id))
+    trimmed = token_ids
+    while trimmed.size(1) > 0 and int(trimmed[0, -1].item()) in stop_ids:
+        trimmed = trimmed[:, :-1]
+    return trimmed
+
+
+def _assistant_turn_boundaries_from_messages(
+    tokenizer,
+    msgs,
+    *,
+    prompt_add_generation_prompt: bool,
+    device,
+):
+    starts: list[int] = []
+    ends: list[int] = []
+
+    for idx, msg in enumerate(msgs):
+        if msg["role"] != "assistant":
+            continue
+        prefix_ids = tokens_from_messages(
+            tokenizer, msgs[:idx], device, add_generation_prompt=False
+        )
+        turn_ids = tokens_from_messages(
+            tokenizer, msgs[: idx + 1], device, add_generation_prompt=False
+        )
+        starts.append(int(prefix_ids.size(1)))
+        ends.append(int(turn_ids.size(1)))
+
+    if bool(prompt_add_generation_prompt) and msgs[-1]["role"] != "assistant":
+        prefix_ids = tokens_from_messages(
+            tokenizer, msgs, device, add_generation_prompt=False
+        )
+        prompt_ids = tokens_from_messages(
+            tokenizer, msgs, device, add_generation_prompt=True
+        )
+        starts.append(int(prefix_ids.size(1)))
+        ends.append(int(prompt_ids.size(1)))
+
+    if not starts:
+        return None, None, None
+
+    starts_tensor = torch.tensor([starts], device=device, dtype=torch.long)
+    ends_tensor = torch.tensor([ends], device=device, dtype=torch.long)
+    start_mask = torch.ones((1, len(starts)), device=device, dtype=torch.bool)
+    return starts_tensor, ends_tensor, start_mask
+
+
+def build_structured_prompt_segments(
+    tokenizer,
+    prompt_messages,
+    *,
+    prompt_add_generation_prompt: bool,
+    device,
+):
+    msgs = normalize_prompt_messages(prompt_messages)
+    if not msgs:
+        raise ValueError("prompt_messages must contain at least one non-empty message.")
+
+    if bool(prompt_add_generation_prompt):
+        if msgs[-1]["role"] == "assistant":
+            raise ValueError(
+                "prompt_add_generation_prompt=True requires prompt_messages to end with a non-assistant role."
+            )
+        prefix_messages = msgs
+        prefix_ids = tokens_from_messages(
+            tokenizer, prefix_messages, device, add_generation_prompt=False
+        )
+        prompt_ids = tokens_from_messages(
+            tokenizer, prefix_messages, device, add_generation_prompt=True
+        )
+        assistant_prefill_ids = prompt_ids[:, prefix_ids.size(1) :]
+    else:
+        if msgs[-1]["role"] != "assistant":
+            raise ValueError(
+                "prompt_add_generation_prompt=False requires prompt_messages to end with an assistant prefix."
+            )
+        if len(msgs) < 2:
+            raise ValueError(
+                "assistant-prefix prompts require at least one preceding non-assistant message."
+            )
+        prefix_messages = msgs[:-1]
+        assistant_text = str(msgs[-1]["content"] or "")
+        prefix_ids = tokens_from_messages(
+            tokenizer, prefix_messages, device, add_generation_prompt=False
+        )
+        prompt_prefix = tokens_from_messages(
+            tokenizer, prefix_messages, device, add_generation_prompt=True
+        )
+        assistant_content_ids = _assistant_content_delta_from_messages(
+            tokenizer,
+            prefix_messages,
+            assistant_text,
+            prompt_prefix,
+            device,
+        )
+        assistant_content_ids = _strip_trailing_assistant_stop_tokens(
+            tokenizer, assistant_content_ids
+        )
+        assistant_header_ids = prompt_prefix[:, prefix_ids.size(1) :]
+        assistant_prefill_ids = torch.cat(
+            [assistant_header_ids, assistant_content_ids], dim=1
+        )
+        prompt_ids = torch.cat([prefix_ids, assistant_prefill_ids], dim=1)
+
+    if assistant_prefill_ids.size(1) <= 0:
+        raise ValueError(
+            "Structured direct LLoPA prompt produced an empty assistant prefill segment."
+        )
+
+    if prefix_messages and prefix_messages[0]["role"] == "system":
+        system_ids = tokens_from_messages(
+            tokenizer, [prefix_messages[0]], device, add_generation_prompt=False
+        )
+        user_ids = prefix_ids[:, system_ids.size(1) :]
+    else:
+        system_ids = prefix_ids[:, :0]
+        user_ids = prefix_ids
+
+    assistant_header_starts, assistant_turn_ends, assistant_header_start_mask = (
+        _assistant_turn_boundaries_from_messages(
+            tokenizer,
+            msgs,
+            prompt_add_generation_prompt=bool(prompt_add_generation_prompt),
+            device=device,
+        )
+    )
+
+    return {
+        "prefix_ids": prefix_ids,
+        "prompt_ids": prompt_ids,
+        "system_ids": system_ids,
+        "user_ids": user_ids,
+        "assistant_prefill_ids": assistant_prefill_ids,
+        "replay_user_prefix_keep_len": 0,
+        "replay_user_start": 0,
+        "replay_user_len": int(user_ids.size(1)),
+        "assistant_header_starts": assistant_header_starts,
+        "assistant_turn_ends": assistant_turn_ends,
+        "assistant_header_start_mask": assistant_header_start_mask,
+    }
+
+
+def _assistant_header_ids_from_chat_template(tokenizer, device):
+    probe_messages_list = [
+        [{"role": "system", "content": "system"}, {"role": "user", "content": "user"}],
+        [{"role": "user", "content": "user"}],
+    ]
+
+    for messages in probe_messages_list:
+        try:
+            rendered_no_prompt = apply_chat_template(
+                tokenizer, messages, add_generation_prompt=False
+            )
+            rendered_with_prompt = apply_chat_template(
+                tokenizer, messages, add_generation_prompt=True
+            )
+        except Exception:
+            continue
+
+        if not isinstance(rendered_no_prompt, str) or not isinstance(
+            rendered_with_prompt, str
+        ):
+            continue
+        if not rendered_with_prompt or rendered_with_prompt == rendered_no_prompt:
+            continue
+
+        if rendered_with_prompt.startswith(rendered_no_prompt):
+            suffix_text = rendered_with_prompt[len(rendered_no_prompt) :]
+            if suffix_text:
+                try:
+                    ids = tokenizer(
+                        suffix_text,
+                        add_special_tokens=False,
+                        return_tensors="pt",
+                    ).input_ids.to(device)
+                    if ids.numel() > 0:
+                        return ids
+                except Exception:
+                    pass
+
+        try:
+            ids_no_prompt = tokens_from_messages(
+                tokenizer, messages, device, add_generation_prompt=False
+            )
+            ids_with_prompt = tokens_from_messages(
+                tokenizer, messages, device, add_generation_prompt=True
+            )
+        except Exception:
+            continue
+
+        if ids_with_prompt.numel() == 0 or ids_with_prompt.size(1) <= ids_no_prompt.size(1):
+            continue
+
+        prefix_len = lcp_len(ids_no_prompt, ids_with_prompt)
+        if prefix_len < ids_with_prompt.size(1):
+            delta_ids = ids_with_prompt[:, prefix_len:]
+            if delta_ids.numel() > 0:
+                return delta_ids
+    return None
+
+
+def assistant_header_ids(tokenizer, device):
+    if _is_mistral_template(tokenizer):
+        try:
+            tok_id = tokenizer.convert_tokens_to_ids(MISTRAL_ASSIST_START)
+            if tok_id is None or tok_id == tokenizer.unk_token_id:
+                return None
+            return torch.tensor([[int(tok_id)]], device=device, dtype=torch.long)
+        except Exception:
+            return None
+    tmpl = getattr(tokenizer, "chat_template", "") or ""
+    if "<|start_header_id|>" in tmpl:
+        header = "<|start_header_id|>assistant<|end_header_id|>\n\n"
+        try:
+            return tokenizer(
+                header, add_special_tokens=False, return_tensors="pt"
+            ).input_ids.to(device)
+        except Exception:
+            return None
+    return _assistant_header_ids_from_chat_template(tokenizer, device)
+
+
+__all__ = [
+    "apply_chat_template",
+    "assistant_header_ids",
+    "build_structured_prompt_segments",
+    "lcp_len",
+    "normalize_prompt_messages",
+    "tokens_from_messages",
+]

modeling_llopa.py

@@ -0,0 +1,99 @@
+# coding: utf-8
+import inspect
+import importlib
+import importlib.util
+import sys
+from pathlib import Path
+import torch
+
+
+_ALIASES = {
+    "attn_implementation": "attn_impl",
+    "_attn_implementation": "attn_impl",
+}
+
+
+def _normalize_dtype(value):
+    if value is None:
+        return None
+    if isinstance(value, str):
+        s = value.strip().lower()
+    elif value is torch.bfloat16:
+        s = "bfloat16"
+    elif value is torch.float16:
+        s = "float16"
+    elif value is torch.float32:
+        s = "float32"
+    else:
+        return None
+    mapping = {
+        "auto": "auto",
+        "bf16": "bf16",
+        "bfloat16": "bf16",
+        "fp16": "fp16",
+        "float16": "fp16",
+        "half": "fp16",
+        "fp32": "fp32",
+        "float32": "fp32",
+    }
+    return mapping.get(s)
+
+
+def _resolve_load_llopa_model(pretrained_model_name_or_path):
+    candidates = []
+    try:
+        src = Path(pretrained_model_name_or_path).expanduser().resolve()
+        if src.is_dir():
+            candidates.append(src / 'llopa_inference.py')
+    except Exception:
+        pass
+    candidates.append(Path(__file__).resolve().parent / 'llopa_inference.py')
+    for infer_path in candidates:
+        if not infer_path.is_file():
+            continue
+        repo_dir = infer_path.parent
+        if str(repo_dir) not in sys.path:
+            sys.path.insert(0, str(repo_dir))
+        spec = importlib.util.spec_from_file_location('llopa_inference_runtime', str(infer_path))
+        if spec is None or spec.loader is None:
+            continue
+        mod = importlib.util.module_from_spec(spec)
+        sys.modules['llopa_inference_runtime'] = mod
+        spec.loader.exec_module(mod)
+        return mod.load_llopa_model
+    try:
+        return importlib.import_module('llopa_inference').load_llopa_model
+    except Exception as exc:
+        raise RuntimeError('Unable to resolve load_llopa_model for local LLOPA package') from exc
+
+
+class LLOPAForCausalLM:
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        # Keep only load_llopa_model kwargs, while supporting HF aliases.
+        load_llopa_model = _resolve_load_llopa_model(pretrained_model_name_or_path)
+        allowed = set(inspect.signature(load_llopa_model).parameters.keys())
+        norm = {}
+        dtype_val = kwargs.get("dtype", kwargs.get("torch_dtype"))
+        for k, v in kwargs.items():
+            kk = _ALIASES.get(k, k)
+            if kk in allowed and kk not in norm:
+                norm[kk] = v
+        if "dtype" in allowed and dtype_val is not None:
+            nd = _normalize_dtype(dtype_val)
+            if nd is not None:
+                norm["dtype"] = nd
+        kwargs = norm
+        # Handle single-device forms users often pass as device_map.
+        dm = kwargs.get("device_map")
+        if isinstance(dm, str):
+            s = dm.strip()
+            if s in ("", "none", "None", "null", "NULL"):
+                kwargs["device_map"] = None
+            elif s in ("cuda", "cpu", "mps") or s.startswith(("cuda:", "xpu:", "npu:")):
+                kwargs.setdefault("device", s)
+                kwargs["device_map"] = None
+        model, _ = load_llopa_model(pretrained_model_name_or_path, **kwargs)
+        return model
+
+__all__ = ["LLOPAForCausalLM"]

prompt_templates.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Optional, Sequence, Tuple, Union
+
+DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant."
+QA_BOXED_INSTRUCTION = "Provide the answer and finish with the short answer inside \\boxed{}."
+
+TASK_ALIASES = {
+    "docqa": "qa_doc",
+    "doc_qa": "qa_doc",
+    "doc-qa": "qa_doc",
+    "summarize": "summary",
+    "summarization": "summary",
+    "coding": "code",
+}
+
+# Known dataset -> task mapping (extend as needed)
+DATASET_TASKS = {
+    "gsm8k": "math",
+    "math": "math",
+    "nemotron_math_v2": "math",
+    "nvidia_nemotron_math_v2": "math",
+    "triviaqa": "qa_doc",
+    "hotpotqa": "qa_doc",
+    "sniah": "qa_doc",
+    "cnn_dailymail": "summary",
+    "mbpp": "code",
+    "opencodeinstruct": "code",
+    "open_code_instruct": "code",
+}
+
+
+def normalize_task_name(task: Optional[str]) -> str:
+    t = str(task or "qa_doc").strip().lower()
+    return TASK_ALIASES.get(t, t or "qa_doc")
+
+
+def task_from_source(source: Optional[str], explicit_task: Optional[str] = None) -> str:
+    if explicit_task:
+        t = normalize_task_name(explicit_task)
+        if t:
+            return t
+    if source:
+        return DATASET_TASKS.get(str(source).strip().lower(), "qa_doc")
+    return "qa_doc"
+
+
+DocumentInput = Optional[Union[str, Sequence[str]]]
+
+
+def _normalize_documents(document: DocumentInput) -> List[str]:
+    if document is None:
+        return []
+    if isinstance(document, str):
+        doc = document.strip()
+        return [doc] if doc else []
+    if isinstance(document, Sequence):
+        docs = []
+        for item in document:
+            if item is None:
+                continue
+            if not isinstance(item, str):
+                item = str(item)
+            item = item.strip()
+            if item:
+                docs.append(item)
+        return docs
+    doc = str(document).strip()
+    return [doc] if doc else []
+
+
+def join_context_blocks(blocks: Iterable[str]) -> str:
+    parts = [b.strip() for b in blocks if isinstance(b, str) and b.strip()]
+    if not parts:
+        return ""
+    if len(parts) == 1:
+        return parts[0]
+    # Join with blank lines only (no numbering labels).
+    return "\n\n".join(parts)
+
+
+def _format_doc_block(docs: List[str], singular_label: str, plural_label: str) -> str:
+    if not docs:
+        return ""
+    joined = join_context_blocks(docs)
+    if not joined:
+        return ""
+    return f"{singular_label}:\n{joined}"
+
+
+def build_prompt_parts(task: str, question: str, document: DocumentInput) -> Tuple[str, str]:
+    t = normalize_task_name(task)
+    q = (question or "").strip()
+    docs = _normalize_documents(document)
+
+    if t == "summary":
+        instruction = q if q else "Summarize the passage."
+        doc_text = _format_doc_block(docs, "Passage", "Passages")
+        return doc_text, instruction
+    if t == "code":
+        if not q:
+            q = "Solve the following problem."
+        return "", f"Write code to solve the following problem:\n{q}"
+    if t == "math":
+        content = q if q else join_context_blocks(docs)
+        if content:
+            return "", f"{content}\n\n{QA_BOXED_INSTRUCTION}"
+        return "", QA_BOXED_INSTRUCTION
+
+    # qa_doc default
+    doc_text = _format_doc_block(docs, "Document", "Documents")
+    query_text = f"Question: {q}" if q else ""
+    if query_text:
+        query_text = f"{query_text}\n{QA_BOXED_INSTRUCTION}"
+    else:
+        query_text = QA_BOXED_INSTRUCTION
+    return doc_text, query_text
+
+
+def build_user_content(task: str, question: str, document: DocumentInput) -> str:
+    doc_text, query_text = build_prompt_parts(task, question, document)
+    if doc_text and query_text:
+        return f"{doc_text}\n\n{query_text}"
+    return doc_text or query_text

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9400df98529060210393c40f08cb127f7c0df584338b3fbfdba8cf82a33c1ade
+size 17210102

tokenizer_config.json

@@ -0,0 +1,2071 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

tri_info.txt

@@ -0,0 +1,13 @@
+attention_gate_mode=off
+lower_k=24
+prefill_mode=lower
+prefill_attn=causal
+system_prefill=no_system
+user_prefill=full
+no_upper_attn=0
+replay_module=none
+replay_per_layers=-1
+last_layer_module=none
+num_suffix_specials=0
+fusion_mode=inband
+capsule_llopa_enabled=1