Add 'load first example' button
Browse files
app.py
CHANGED
|
@@ -154,6 +154,20 @@ def format_stats_display(judgment_times, num_judgments, num_skips):
|
|
| 154 |
|
| 155 |
return stats
|
| 156 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
def handle_left_better(prompt, completion_1_display, completion_2_display, current_idx, instruction, completion_1, completion_2,
|
| 158 |
start_time, judgment_times, num_judgments, num_skips):
|
| 159 |
"""Handle when user selects left completion as better"""
|
|
@@ -173,15 +187,15 @@ def handle_left_better(prompt, completion_1_display, completion_2_display, curre
|
|
| 173 |
# Get new random example
|
| 174 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
| 175 |
|
|
|
|
|
|
|
|
|
|
| 176 |
# Update stats display
|
| 177 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
| 178 |
|
| 179 |
message = "β
Annotation saved! Left completion selected as better." if success else "β
Left completion selected (save failed - check console)"
|
| 180 |
gr.Info(message)
|
| 181 |
|
| 182 |
-
# Reset timer for new example
|
| 183 |
-
new_start_time = time.time()
|
| 184 |
-
|
| 185 |
return (
|
| 186 |
new_prompt,
|
| 187 |
new_comp_1,
|
|
@@ -216,15 +230,15 @@ def handle_right_better(prompt, completion_1_display, completion_2_display, curr
|
|
| 216 |
# Get new random example
|
| 217 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
| 218 |
|
|
|
|
|
|
|
|
|
|
| 219 |
# Update stats display
|
| 220 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
| 221 |
|
| 222 |
message = "β
Annotation saved! Right completion selected as better." if success else "β
Right completion selected (save failed - check console)"
|
| 223 |
gr.Info(message)
|
| 224 |
|
| 225 |
-
# Reset timer for new example
|
| 226 |
-
new_start_time = time.time()
|
| 227 |
-
|
| 228 |
return (
|
| 229 |
new_prompt,
|
| 230 |
new_comp_1,
|
|
@@ -280,11 +294,23 @@ if HF_TOKEN:
|
|
| 280 |
else:
|
| 281 |
print("Warning: No HF_TOKEN found. Annotations will not be saved.")
|
| 282 |
|
| 283 |
-
|
| 284 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
# Create Gradio interface
|
| 287 |
-
|
|
|
|
| 288 |
gr.Markdown(f"""
|
| 289 |
# π― AI Alignment: Binary Preference Annotation
|
| 290 |
|
|
@@ -297,29 +323,34 @@ This simulates the data annotation process used in RLHF (Reinforcement Learning
|
|
| 297 |
""")
|
| 298 |
|
| 299 |
# State to track current example and its components
|
| 300 |
-
current_idx = gr.State(
|
| 301 |
-
current_instruction = gr.State(
|
| 302 |
-
current_completion_1 = gr.State(
|
| 303 |
-
current_completion_2 = gr.State(
|
| 304 |
|
| 305 |
# State to track timing and statistics
|
| 306 |
-
start_time = gr.State(
|
| 307 |
judgment_times = gr.State([]) # List of times taken for each judgment
|
| 308 |
num_judgments = gr.State(0) # Number of judgments made
|
| 309 |
num_skips = gr.State(0) # Number of examples skipped
|
| 310 |
|
| 311 |
-
#
|
| 312 |
-
|
|
|
|
|
|
|
|
|
|
| 313 |
|
| 314 |
-
# Display completions side by side
|
| 315 |
-
|
|
|
|
| 316 |
with gr.Column():
|
| 317 |
-
completion_1_display = gr.Markdown(
|
| 318 |
with gr.Column():
|
| 319 |
-
completion_2_display = gr.Markdown(
|
| 320 |
|
| 321 |
-
# Action buttons
|
| 322 |
-
|
|
|
|
| 323 |
left_better_btn = gr.Button("π Left is Better", variant="primary", size="lg")
|
| 324 |
skip_btn = gr.Button("βοΈ Skip This Example", variant="secondary", size="lg")
|
| 325 |
right_better_btn = gr.Button("π Right is Better", variant="primary", size="lg")
|
|
@@ -333,7 +364,15 @@ This simulates the data annotation process used in RLHF (Reinforcement Learning
|
|
| 333 |
# Statistics display
|
| 334 |
stats_display = gr.Markdown("π **Session Statistics:** No judgments made yet.", label="Performance Stats")
|
| 335 |
|
| 336 |
-
# Wire up the
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
left_better_btn.click(
|
| 338 |
handle_left_better,
|
| 339 |
inputs=[prompt_display, completion_1_display, completion_2_display, current_idx, current_instruction, current_completion_1, current_completion_2,
|
|
|
|
| 154 |
|
| 155 |
return stats
|
| 156 |
|
| 157 |
+
def load_first_example():
|
| 158 |
+
"""Load the first example and start the annotation interface"""
|
| 159 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2 = get_random_example()
|
| 160 |
+
start_time = time.time()
|
| 161 |
+
|
| 162 |
+
return (
|
| 163 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2,
|
| 164 |
+
start_time,
|
| 165 |
+
gr.update(visible=False), # Hide load button
|
| 166 |
+
gr.update(visible=True), # Show prompt
|
| 167 |
+
gr.update(visible=True), # Show completion row
|
| 168 |
+
gr.update(visible=True), # Show action buttons
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
def handle_left_better(prompt, completion_1_display, completion_2_display, current_idx, instruction, completion_1, completion_2,
|
| 172 |
start_time, judgment_times, num_judgments, num_skips):
|
| 173 |
"""Handle when user selects left completion as better"""
|
|
|
|
| 187 |
# Get new random example
|
| 188 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
| 189 |
|
| 190 |
+
# Reset timer for new example
|
| 191 |
+
new_start_time = time.time()
|
| 192 |
+
|
| 193 |
# Update stats display
|
| 194 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
| 195 |
|
| 196 |
message = "β
Annotation saved! Left completion selected as better." if success else "β
Left completion selected (save failed - check console)"
|
| 197 |
gr.Info(message)
|
| 198 |
|
|
|
|
|
|
|
|
|
|
| 199 |
return (
|
| 200 |
new_prompt,
|
| 201 |
new_comp_1,
|
|
|
|
| 230 |
# Get new random example
|
| 231 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
| 232 |
|
| 233 |
+
# Reset timer for new example
|
| 234 |
+
new_start_time = time.time()
|
| 235 |
+
|
| 236 |
# Update stats display
|
| 237 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
| 238 |
|
| 239 |
message = "β
Annotation saved! Right completion selected as better." if success else "β
Right completion selected (save failed - check console)"
|
| 240 |
gr.Info(message)
|
| 241 |
|
|
|
|
|
|
|
|
|
|
| 242 |
return (
|
| 243 |
new_prompt,
|
| 244 |
new_comp_1,
|
|
|
|
| 294 |
else:
|
| 295 |
print("Warning: No HF_TOKEN found. Annotations will not be saved.")
|
| 296 |
|
| 297 |
+
def load_first_example():
|
| 298 |
+
"""Load the first example and start the annotation interface"""
|
| 299 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2 = get_random_example()
|
| 300 |
+
start_time = time.time()
|
| 301 |
+
|
| 302 |
+
return (
|
| 303 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2,
|
| 304 |
+
start_time,
|
| 305 |
+
gr.update(visible=False), # Hide load button
|
| 306 |
+
gr.update(visible=True), # Show prompt
|
| 307 |
+
gr.update(visible=True), # Show completion row
|
| 308 |
+
gr.update(visible=True), # Show action buttons
|
| 309 |
+
)
|
| 310 |
|
| 311 |
# Create Gradio interface
|
| 312 |
+
|
| 313 |
+
with gr.Blocks(title="AI Alignment: Binary Preference Annotation", css=".square-button { height: 80px !important; }") as demo:
|
| 314 |
gr.Markdown(f"""
|
| 315 |
# π― AI Alignment: Binary Preference Annotation
|
| 316 |
|
|
|
|
| 323 |
""")
|
| 324 |
|
| 325 |
# State to track current example and its components
|
| 326 |
+
current_idx = gr.State(0)
|
| 327 |
+
current_instruction = gr.State("")
|
| 328 |
+
current_completion_1 = gr.State("")
|
| 329 |
+
current_completion_2 = gr.State("")
|
| 330 |
|
| 331 |
# State to track timing and statistics
|
| 332 |
+
start_time = gr.State(0.0) # When current example was loaded
|
| 333 |
judgment_times = gr.State([]) # List of times taken for each judgment
|
| 334 |
num_judgments = gr.State(0) # Number of judgments made
|
| 335 |
num_skips = gr.State(0) # Number of examples skipped
|
| 336 |
|
| 337 |
+
# Load first example button (shown initially)
|
| 338 |
+
load_first_btn = gr.Button("π Load First Example", variant="primary", size="lg", elem_classes="square-button")
|
| 339 |
+
|
| 340 |
+
# Display prompt (hidden initially)
|
| 341 |
+
prompt_display = gr.Markdown("", label="Prompt", visible=False)
|
| 342 |
|
| 343 |
+
# Display completions side by side (hidden initially)
|
| 344 |
+
completion_row = gr.Row(visible=False)
|
| 345 |
+
with completion_row:
|
| 346 |
with gr.Column():
|
| 347 |
+
completion_1_display = gr.Markdown("", label="Completion A (Left)")
|
| 348 |
with gr.Column():
|
| 349 |
+
completion_2_display = gr.Markdown("", label="Completion B (Right)")
|
| 350 |
|
| 351 |
+
# Action buttons (hidden initially)
|
| 352 |
+
action_buttons = gr.Row(visible=False)
|
| 353 |
+
with action_buttons:
|
| 354 |
left_better_btn = gr.Button("π Left is Better", variant="primary", size="lg")
|
| 355 |
skip_btn = gr.Button("βοΈ Skip This Example", variant="secondary", size="lg")
|
| 356 |
right_better_btn = gr.Button("π Right is Better", variant="primary", size="lg")
|
|
|
|
| 364 |
# Statistics display
|
| 365 |
stats_display = gr.Markdown("π **Session Statistics:** No judgments made yet.", label="Performance Stats")
|
| 366 |
|
| 367 |
+
# Wire up the load first example button
|
| 368 |
+
load_first_btn.click(
|
| 369 |
+
load_first_example,
|
| 370 |
+
inputs=[],
|
| 371 |
+
outputs=[prompt_display, completion_1_display, completion_2_display, current_idx, current_instruction, current_completion_1, current_completion_2,
|
| 372 |
+
start_time, load_first_btn, prompt_display, completion_row, action_buttons]
|
| 373 |
+
)
|
| 374 |
+
|
| 375 |
+
# Wire up the action buttons
|
| 376 |
left_better_btn.click(
|
| 377 |
handle_left_better,
|
| 378 |
inputs=[prompt_display, completion_1_display, completion_2_display, current_idx, current_instruction, current_completion_1, current_completion_2,
|