tweak

app.py

@@ -1,10 +1,132 @@
 import gradio as gr
 import os
 import sys
+import json
 from PIL import Image,ImageDraw
+classes = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'tick', 'fraction']
+API_KEY = os.environ.get("ROBOFLOW_API_KEY")
+def packFilterBoxInfo(filter_box_info):
+    # 数字类别映射
+    digit_classes = {
+        'one': '1', 'two': '2', 'three': '3', 'four': '4', 'five': '5',
+        'six': '6', 'seven': '7', 'eight': '8', 'nine': '9', 'zero': '0'
+    }
+    fraction_boxes = []
+    number_boxes = []
+    
+    for box in filter_box_info:
+        if box['class'] == 'fraction':
+            fraction_boxes.append(box)
+        elif box['class'] in digit_classes:
+            number_boxes.append(box)
+    
+    fraction_boxes.sort(key=lambda x: x['x'] - x['width']/2)
 
+    fraction_values = []
+
+    for frac_box in fraction_boxes:
+        # fraction框的边界
+        frac_x = frac_box['x']
+        frac_y = frac_box['y']
+        frac_width = frac_box['width']
+        frac_height = frac_box['height']
+        
+        # 定义分子分母的区域
+        numerator_numbers = []
+        denominator_numbers = []
+        
+        # 遍历所有数字，判断是否在当前fraction框内
+        for num_box in number_boxes:
+            # 检查数字是否在fraction框的水平范围内
+            if (frac_x - frac_width/2 <= num_box['x'] <= frac_x + frac_width/2 and frac_y - frac_height/2 <= num_box['y'] <= frac_y + frac_height/2):
+                # 获取数字值
+                digit = digit_classes[num_box['class']]
+                
+                # 根据y坐标判断是分子还是分母
+                if num_box['y'] < frac_y:  # 在分数线上方
+                    numerator_numbers.append((num_box['x'], num_box['y'], num_box['width'], num_box['height'], digit))
+                else:  # 在分数线下方
+                    denominator_numbers.append((num_box['x'], num_box['y'], num_box['width'], num_box['height'], digit))
+        
+        # 按x坐标排序
+        numerator_numbers.sort(key=lambda x: x[0]-x[2]/2)
+        denominator_numbers.sort(key=lambda x: x[0]-x[2]/2)
+        
+        # 提取排序后的数字
+        numerator = ''.join(digit for _, _, _, _, digit in numerator_numbers)
+        denominator = ''.join(digit for _, _, _, _, digit in denominator_numbers)
+        if numerator == "": 
+            numerator = "?"
+        if denominator == "":
+            denominator = "?"
+        fraction_values.append(f"{numerator}/{denominator}")
+        
+    return fraction_values
+#Assume its coordinate are top-left, bottom-right    
+def getOverlap(box1, box2):
+    b1_x1, b1_y1, b1_x2, b1_y2 = box1
+    b2_x1, b2_y1, b2_x2, b2_y2 = box2
+    inter_x1 = max(b1_x1, b2_x1)
+    inter_y1= max(b1_y1, b2_y1)
+    inter_x2 = min(b1_x2, b2_x2)
+    inter_y2 = min(b1_y2, b2_y2)
+    if inter_x1 < inter_x2 and inter_y1 < inter_y2:
+        inter_area = (inter_x2 - inter_x1) * (inter_y2 - inter_y1)
+    else:
+        return 0.0
+    b1_area = abs(b1_x2 - b1_x1) * abs(b1_y2 - b1_y1)
+    b2_area = abs(b2_x2 - b2_x1) * abs(b2_y2 - b2_y1)
+    union_area = b2_area + b1_area - inter_area
+
+    return inter_area / union_area
+
+def generate_textual_description(box_info):
+    fraction_values = packFilterBoxInfo(box_info)
+    # Create a dictionary to store information by class ID
+    class_summary = {c: [] for c in classes}
+    for box in box_info:
+        c_name = box['class']
+        if c_name not in class_summary:
+            continue
+        else:
+            x, y, w, h = box['x'], box['y'], box['width'], box['height']
+            class_summary[c_name].append([x-w/2, y-h/2, x+w/2, y+h/2])
+    # Generate a summary for each class
+    #the index of the left one
+    class_summary['zero'].sort()
+    left_most_zero_cor = class_summary['zero'][0]
+    left_zero = True 
+    class_summary['one'].sort()
+    right_most_one_cor = class_summary['one'][-1]
+    right_one = True
+    for fra_box in class_summary['fraction']:
+        if getOverlap(fra_box, left_most_zero_cor) >= 0.5:
+            left_zero = False
+        if getOverlap(fra_box, right_most_one_cor) >= 0.5:
+            left_one = False
+    textual_description = ""
+    textual_description += "The key elements are interpreted via visual translator. Their coordinates are represented as outlined boxes (top-left, bottom-right)"
+    #print(f"The key elements are interpreted via visual translator. Their coordinates are represented as outlined boxes (top-left, bottom-right)")
+    if left_zero:
+        textual_description += f"There is a zero on the left side of the number line. Its coordinate is ({(left_most_zero_cor[0]:.2f, left_most_zero_cor[1]:.2f), (left_most_zero_cor[2]:.2f, left_most_zero_cor[3]:.2f)})"
+    if right_one:
+        textual_description += f"There is a one on the right side of the number line. Its coordinate is ({(right_most_one_cor[0]:.2f, right_most_one_cor[1]:.2f), (right_most_one_cor[2]:.2f, right_most_one_cor[3]:.2f)})"
+    present_classes = ['fraction', 'tick']
+    for cid, boxes in class_summary.items():
+        class_name = cid
+        if class_name not in present_classes:
+            continue
+        count = len(boxes)
+        boxes.sort(key=lambda x: x[0]) # it has been the x of the top-left corner
+        if count > 0:
+            textual_description += f"\nThere are {count} {class_name}s. Their coordinates are: "
+            for box in boxes:
+                textual_description += f"{{({box[0]:.2f}, {box[1]:.2f}), ({box[2]:.2f}, {box[3]:.2f})}, }" 
+            if (class_name == "fraction"):
+                textual_description += f"\nThe fraction numbers from left to right are: {fraction_values}. "
+    return textual_description
 def drawWithAllBox_info(pil_image, box_info):
-    colors = ['red', 'green', 'blue', 'orange', 'purple', 'cyan', 'magenta', 'yellow', 'brown', 'pink', 'gray', 'lime']
+    colors = ['red', 'green', 'blue', 'orange', 'purple', 'cyan', 'magenta', 'yellow', 'brown', 'pink', 'gray', 'lime', 'navy']
     draw = ImageDraw.Draw(pil_image)
     for box in box_info:
         x, y, w, h = box['x'], box['y'], box['width'], box['height']
@@ -27,12 +149,52 @@ def draw_image_with_boxes(image):
     pil_image = image.copy() if hasattr(image, 'copy') else Image.fromarray(image)
     return drawWithAllBox_info(pil_image, test_box_info)
 
-demo = gr.Interface(
-    fn=draw_image_with_boxes,
-    inputs=gr.Image(type="pil", label="Upload Image"),
-    outputs=gr.Image(type="pil", label="Image with Boxes"),
-    title="Fraction & Tick Detector",
-    description="Upload an image and see the detected boxes drawn on it."
-)
+def process_image(image):
+    if image is None:
+        return None, "", None
+    pil_image = image.copy() if hasattr(image, 'copy') else Image.fromarray(image)
+    boxed_img = drawWithAllBox_info(pil_image, test_box_info)
+    textual = generate_textual_description(test_box_info)
+    # kept_box_info assumed to be test_box_info for this demo
+    json_str = json.dumps(test_box_info, indent=2)
+    return boxed_img, textual, json_str
+
+def download_json(json_str):
+    # Save JSON to a temp file for download
+    import tempfile
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".json", mode="w") as f:
+        f.write(json_str)
+        temp_path = f.name
+    return gr.File.update(value=temp_path, visible=True)
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Fraction & Tick Detector\nUpload an image to see detected boxes, textual description, and download the box info as JSON.")
+    with gr.Row():
+        with gr.Column():
+            img_input = gr.Image(type="pil", label="Upload Image")
+            run_btn = gr.Button("Run Detection")
+        with gr.Column():
+            img_out = gr.Image(type="pil", label="Image with Boxes")
+            text_out = gr.Textbox(label="Textual Description", lines=8)
+            json_file = gr.File(label="Download Box Info (JSON)", visible=False)
+            download_btn = gr.Button("Download Box Info as JSON")
+    # State to hold the JSON string
+    json_state = gr.State("")
+
+    def _process(image):
+        boxed_img, textual, json_str = process_image(image)
+        return boxed_img, textual, json_str, gr.File.update(visible=False)
+
+    run_btn.click(_process, inputs=img_input, outputs=[img_out, text_out, json_state, json_file])
+
+    def _download(json_str):
+        # Save JSON to a temp file for download
+        import tempfile
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".json", mode="w") as f:
+            f.write(json_str)
+            temp_path = f.name
+        return gr.File.update(value=temp_path, visible=True)
+
+    download_btn.click(_download, inputs=json_state, outputs=json_file)
 
 demo.launch()