Update app.py

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import torch
 import transformers
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from PIL import Image
 import warnings
 
@@ -14,19 +14,12 @@ warnings.filterwarnings('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(f"Using device: {device}")
 
-model_name = 'cognitivecomputations/dolphin-vision-72b'
+model_name = 'cognitivecomputations/dolphin-vision-7b'
 
-# Configure 8-bit quantization
-quantization_config = BitsAndBytesConfig(
-    load_in_8bit=True,
-    llm_int8_threshold=6.0,
-    llm_int8_has_fp16_weight=False
-)
-
-# create model and load it to the specified device with 8-bit quantization
+# create model and load it to the specified device
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
-    quantization_config=quantization_config,
+    torch_dtype=torch.float16,
     device_map="auto",  # This will automatically use the GPU if available
     trust_remote_code=True
 )
@@ -57,7 +50,7 @@ def inference(prompt, image):
     print(f"Device of image_tensor: {image_tensor.device}")
 
     # generate
-    with torch.inference_mode():
+    with torch.cuda.amp.autocast():
         output_ids = model.generate(
             input_ids,
             images=image_tensor,