| { |
| "_name_or_path": "", |
| "architectures": [ |
| "MultiModalLLM_PT" |
| ], |
| "auto_map": { |
| "AutoConfig": "model_config.VideoChatEConfig", |
| "AutoModel": "modeling_videochate.MultiModalLLM_PT" |
| }, |
| "model_config": { |
| "bridge": { |
| "extra_num_query_token": 64, |
| "name": "qformer", |
| "num_query_token": 32, |
| "qformer_attention_probs_dropout_prob": 0.1, |
| "qformer_drop_path_rate": 0.2, |
| "qformer_hidden_dropout_prob": 0.1 |
| }, |
| "freeze_bridge": false, |
| "freeze_llm": false, |
| "freeze_vision_encoder": false, |
| "llm": { |
| "lora_alpha": 32, |
| "lora_dropout": 0.1, |
| "lora_r": 16, |
| "name": "mistral_7b", |
| "pretrained_llm_path": "mistralai/Mistral-7B-Instruct-v0.3", |
| "use_lora": true, |
| "hidden_size": 4096 |
| }, |
| "loss": { |
| "use_vision_regression_loss": false |
| }, |
| "pretrained_paths": {}, |
| |
| "vision_encoder": { |
| "name":"vit_l14", |
| "img_size":224, |
| "patch_size":16, |
| "d_model":1024, |
| "encoder_embed_dim":1024, |
| "encoder_depth":24, |
| "encoder_num_heads":16, |
| "drop_path_rate": 0.0, |
| "num_frames":16, |
| "tubelet_size":1, |
| "use_checkpoint":false, |
| "checkpoint_num":0, |
| "return_index":-2, |
| "vit_add_ln":true, |
| "pretrained": null |
| } |
| }, |
| "torch_dtype": "float32", |
| "transformers_version": "4.38.0", |
| "use_flash_attention": true, |
| "use_cache": true, |
| "build_decoder":true, |
| "hidden_size": 4096 |
| } |
|
|
