| |
| import os |
| import json |
|
|
| import torch |
| from transformers import LlamaTokenizer, LlamaForCausalLM |
|
|
| tokenizer = LlamaTokenizer.from_pretrained("../7B-2nd-train") |
|
|
| base_model = LlamaForCausalLM.from_pretrained( |
| "../7B-2nd-train", |
| load_in_8bit=False, |
| torch_dtype=torch.float16, |
| device_map={"": "cpu"}, |
| ) |
|
|
| base_model_sd = base_model.state_dict() |
|
|
| params = { |
| "dim": 4096, |
| "multiple_of": 256, |
| "n_heads": 32, |
| "n_layers": 32, |
| "norm_eps": 1e-06, |
| "vocab_size": -1, |
| } |
| n_layers = params["n_layers"] |
| n_heads = params["n_heads"] |
| dim = params["dim"] |
| dims_per_head = dim // n_heads |
| base = 10000.0 |
| inv_freq = 1.0 / \ |
| (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head)) |
|
|
|
|
| def permute(w): |
| return ( |
| w.view(n_heads, dim // n_heads // 2, 2, |
| dim).transpose(1, 2).reshape(dim, dim) |
| ) |
|
|
|
|
| def unpermute(w): |
| return ( |
| w.view(n_heads, 2, dim // n_heads // 2, |
| dim).transpose(1, 2).reshape(dim, dim) |
| ) |
|
|
|
|
| def translate_state_dict_key(k): |
| k = k.replace("base_model.model.", "") |
| if k == "model.embed_tokens.weight": |
| return "tok_embeddings.weight" |
| elif k == "model.norm.weight": |
| return "norm.weight" |
| elif k == "lm_head.weight": |
| return "output.weight" |
| elif k.startswith("model.layers."): |
| layer = k.split(".")[2] |
| if k.endswith(".self_attn.q_proj.weight"): |
| return f"layers.{layer}.attention.wq.weight" |
| elif k.endswith(".self_attn.k_proj.weight"): |
| return f"layers.{layer}.attention.wk.weight" |
| elif k.endswith(".self_attn.v_proj.weight"): |
| return f"layers.{layer}.attention.wv.weight" |
| elif k.endswith(".self_attn.o_proj.weight"): |
| return f"layers.{layer}.attention.wo.weight" |
| elif k.endswith(".mlp.gate_proj.weight"): |
| return f"layers.{layer}.feed_forward.w1.weight" |
| elif k.endswith(".mlp.down_proj.weight"): |
| return f"layers.{layer}.feed_forward.w2.weight" |
| elif k.endswith(".mlp.up_proj.weight"): |
| return f"layers.{layer}.feed_forward.w3.weight" |
| elif k.endswith(".input_layernorm.weight"): |
| return f"layers.{layer}.attention_norm.weight" |
| elif k.endswith(".post_attention_layernorm.weight"): |
| return f"layers.{layer}.ffn_norm.weight" |
| elif k.endswith("rotary_emb.inv_freq") or "lora" in k: |
| return None |
| else: |
| print(layer, k) |
| raise NotImplementedError |
| else: |
| print(k) |
| raise NotImplementedError |
|
|
|
|
| new_state_dict = {} |
| for k, v in base_model_sd.items(): |
| new_k = translate_state_dict_key(k) |
| if new_k is not None: |
| if "wq" in new_k or "wk" in new_k: |
| new_state_dict[new_k] = unpermute(v) |
| else: |
| new_state_dict[new_k] = v |
|
|
| torch.save(new_state_dict, "consolidated.00.pth") |
|
|
| with open("params.json", "w") as f: |
| json.dump(params, f) |
|
|
| |
| model = torch.load("consolidated.00.pth", map_location=torch.device('cpu')) |
| x = model["tok_embeddings.weight"] |
| y = model["output.weight"] |
| row_exclude = 32000 |
| x = x[:row_exclude] |
| y = y[:row_exclude] |
| model["tok_embeddings.weight"] = x |
| model["output.weight"] = y |
| torch.save(model, "consolidated.01.pth") |
| |
|
|
