app.py

import json

from KMVE_RG.models.SGF_model import SGF
from KMVE_RG.modules.tokenizers import Tokenizer
from KMVE_RG.modules.metrics import compute_scores
import numpy as np
from utils.thyroid_gen_config import config as thyroid_args
from utils.liver_gen_config import config as liver_args
from utils.breast_gen_config import config as breast_args

import gradio as gr
import torch
from PIL import Image
import os
from torchvision import transforms

np.random.seed(9233)
torch.manual_seed(9233)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

class Generator(object):
    def __init__(self, model_type):
        if model_type == '甲状腺':
            self.args = thyroid_args
        elif model_type == '乳腺':
            self.args = breast_args
        elif model_type == '肝脏':
            self.args = liver_args
        self.tokenizer = Tokenizer(self.args)
        self.model = SGF(self.args, self.tokenizer)
        sd = torch.load(self.args.models)['state_dict']
        msg = self.model.load_state_dict(sd)
        print(msg)
        self.model.eval()
        self.metrics = compute_scores
        self.transform = transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize((0.485, 0.456, 0.406),
                                 (0.229, 0.224, 0.225))])
        with open(self.args.ann_path, 'r', encoding='utf-8-sig') as f:
            self.data = json.load(f)
        print('模型加载完成')

    def image_process(self, img_paths):
        image_1 = Image.open(os.path.join(self.args.image_dir, img_paths[0])).convert('RGB')
        image_2 = Image.open(os.path.join(self.args.image_dir, img_paths[1])).convert('RGB')
        if self.transform is not None:
            image_1 = self.transform(image_1)
            image_2 = self.transform(image_2)
        image = torch.stack((image_1, image_2), 0)
        return image

    def generate(self, uid):
        img_paths, report = self.data[uid]['img_paths'], self.data[uid]['report']
        imgs = self.image_process(img_paths)
        imgs = imgs.unsqueeze(0)
        with torch.no_grad():
            output, _ = self.model(imgs, mode='sample')
        pred = self.tokenizer.decode(output[0].cpu().numpy())
        gt = self.tokenizer.decode(self.tokenizer(report[:self.args.max_seq_length])[1:])
        scores = self.metrics({0: [gt]}, {0: [pred]})
        return pred, gt, scores

    def visualize_images(self, uid):
        image_1 = Image.open(os.path.join(self.args.image_dir, self.data[uid]['img_paths'][0])).convert('RGB')
        image_2 = Image.open(os.path.join(self.args.image_dir, self.data[uid]['img_paths'][1])).convert('RGB')
        return image_1, image_2

# 主应用程序
def demo():
    with gr.Blocks() as app:
        gr.Markdown("# 超声报告生成Demo")
        gr.Markdown('### SIAT认知与交互技术中心')
        gr.Markdown('### 项目主页：https://lijunrio.github.io/Ultrasound-Report-Generation/')

        # 选择模型
        with gr.Row():
            model_choice = gr.Radio(choices=["甲状腺", "乳腺", "肝脏"], label="请选择模型类型", interactive=True)

        model = gr.State()

        # 展示UID按钮
        uids = [f"uid_{i}" for i in range(20)]
        with gr.Row():
            uid_choice = gr.Radio(choices=[f"{uid}" for uid in uids], label="请选择uid", interactive=False)

        # 定义展示图片的组件
        with gr.Row():
            image1_display = gr.Image(label="图像1", visible=True)
            image2_display = gr.Image(label="图像2", visible=True)

        # 定义生成报告的按钮和文本框
        generate_button = gr.Button("生成报告", interactive=False)
        generated_report_display = gr.Textbox(label="生成的报告", visible=True)
        ground_truth_display = gr.Textbox(label="Ground Truth报告", visible=True)
        nlp_score_display = gr.Textbox(label="NLP得分", visible=True)

        # 加载模型的回调函数
        def load_model_and_uids(model_type):
            model = Generator(model_type)
            return model, gr.update(interactive=True)

        # 点击UID按钮后加载对应的图片
        def on_uid_click(model, uid):
            image1, image2 = model.visualize_images(uid)
            # 显示图片和生成按钮
            return image1, image2, gr.update(interactive=True)

        # 点击生成按钮生成报告
        def on_generate_click(model, uid):
            generated_report, ground_truth_report, nlp_score = model.generate(uid)
            # 展示生成的报告、Ground Truth 和 NLP 得分
            return generated_report, ground_truth_report, f"NLP得分: {nlp_score}"

        # 链接模型选择与UID按钮显示
        model_choice.change(load_model_and_uids, inputs=model_choice, outputs=[model, uid_choice])

        # 链接UID按钮点击与图片显示
        uid_choice.change(on_uid_click, inputs=[model, uid_choice], outputs=[image1_display, image2_display, generate_button])

        generate_button.click(on_generate_click, inputs=[model, uid_choice], outputs=[generated_report_display, ground_truth_display, nlp_score_display])

    return app

if __name__ == '__main__':
    # 启动应用程序
    demo().launch()