update

app.py

@@ -40,13 +40,14 @@ def main():
 
     deprecated_model_handler = DeprecatedModelHandler()
     initial_metric = "ndcg_at_5"
+    initial_metric_v3 = "ndcg_at_10"
 
     # Get pipeline evaluation results
     pipeline_handler = PipelineHandler()
     pipeline_handler.get_pipeline_data()
     initial_language = "overall"
-    data_pipeline = pipeline_handler.render_df(initial_metric, initial_language)
-    data_pipeline = add_rank_and_format(data_pipeline, benchmark_version=3)
+    data_pipeline = pipeline_handler.render_df(initial_metric_v3, initial_language)
+    data_pipeline = add_rank_and_format(data_pipeline, benchmark_version=3, is_pipeline=True)
     
     num_datasets_pipeline = len(data_pipeline.columns) - 4  # Excluding Rank, Model, QPS, Average
     num_scores_pipeline = len(data_pipeline) * num_datasets_pipeline
@@ -133,7 +134,7 @@ def main():
                 ></iframe>
                 """
                 )
-            with gr.TabItem("ViDoRe V3 (Pipeline Eval)"):
+            with gr.TabItem("ViDoRe V3 (Pipeline)"):
                 gr.Markdown("# ViDoRe V3 (Pipeline Evaluation): Retrieval Performance for complex pipelines🔍⚙️")
                 gr.Markdown("### Complete pipeline evaluation including compute costs and timing metrics")
 
@@ -157,7 +158,7 @@ def main():
                             language_choices.append((lang.capitalize(), lang))
 
                     with gr.Row():
-                        metric_dropdown_pipeline = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
+                        metric_dropdown_pipeline = gr.Dropdown(choices=METRICS, value=initial_metric_v3, label="Select Metric")
                         language_dropdown_pipeline = gr.Dropdown(
                             choices=language_choices, 
                             value="overall", 
@@ -191,7 +192,7 @@ def main():
                     def update_data_pipeline(metric, language, search_term, selected_columns):
                         pipeline_handler.get_pipeline_data()
                         data = pipeline_handler.render_df(metric, language)
-                        data = add_rank_and_format(data, benchmark_version=3, selected_columns=selected_columns)
+                        data = add_rank_and_format(data, benchmark_version=3, selected_columns=selected_columns, is_pipeline=True)
                         data = filter_models(data, search_term)
                         if selected_columns:
                             # Include core columns plus selected dataset columns
@@ -206,7 +207,8 @@ def main():
                         refresh_button_pipeline.click(
                             lambda metric, language: add_rank_and_format(
                                 pipeline_handler.render_df(metric, language), 
-                                benchmark_version=3
+                                benchmark_version=3,
+                                is_pipeline=True
                             ),
                             inputs=[metric_dropdown_pipeline, language_dropdown_pipeline],
                             outputs=dataframe_pipeline,
@@ -224,7 +226,7 @@ def main():
                     def refresh_pipeline_data(metric, language):
                         """Refresh pipeline data when metric or language changes."""
                         df = pipeline_handler.render_df(metric, language)
-                        return add_rank_and_format(df, benchmark_version=3)
+                        return add_rank_and_format(df, benchmark_version=3, is_pipeline=True)
                     
                     metric_dropdown_pipeline.change(
                         refresh_pipeline_data,

app/utils.py

@@ -1,7 +1,14 @@
 
 
-def make_clickable_model(model_name, link=None):
-    if link is None:
+def make_clickable_model(model_name, link=None, is_pipeline=False):
+    if is_pipeline:
+        # For pipelines: keep underscores as-is, only process __ and -thisisapoint-
+        desanitized_model_name = model_name.replace("__", "/")
+        desanitized_model_name = desanitized_model_name.replace("-thisisapoint-", ".")
+        if link is None:
+            link = f"https://github.com/illuin-tech/vidore-benchmark/blob/vidore_v3_pipeline/results/pipeline_descriptions/{desanitized_model_name}/description.json"
+    else:
+        # For regular models: replace __ and _ with /, and -thisisapoint- with .
         desanitized_model_name = model_name.replace("__", "/")
         desanitized_model_name = desanitized_model_name.replace("_", "/")
         desanitized_model_name = desanitized_model_name.replace("-thisisapoint-", ".")
@@ -11,7 +18,8 @@ def make_clickable_model(model_name, link=None):
         if "/ocr" in desanitized_model_name:
             desanitized_model_name = desanitized_model_name.replace("/ocr", "")
 
-        link = "https://huggingface.co/" + desanitized_model_name
+        if link is None:
+            link = "https://huggingface.co/" + desanitized_model_name
 
     return f'<a target="_blank" style="text-decoration: underline" href="{link}">{desanitized_model_name}</a>'
 
@@ -51,11 +59,11 @@ def add_rank(df, benchmark_version=1, selected_columns=None):
         return df
 
 
-def add_rank_and_format(df, benchmark_version=1, selected_columns=None):
+def add_rank_and_format(df, benchmark_version=1, selected_columns=None, is_pipeline=False):
     df = df.reset_index()
     df = df.rename(columns={"index": "Model"})
     df = add_rank(df, benchmark_version, selected_columns)
-    df["Model"] = df["Model"].apply(make_clickable_model)
+    df["Model"] = df["Model"].apply(lambda x: make_clickable_model(x, is_pipeline=is_pipeline))
     # df = remove_duplicates(df)
     return df
 
@@ -92,7 +100,7 @@ def get_pipeline_refresh_function(pipeline_handler):
     def _refresh(metric):
         pipeline_handler.get_pipeline_data()
         data = pipeline_handler.render_df(metric)
-        df = add_rank_and_format(data, benchmark_version=3)
+        df = add_rank_and_format(data, benchmark_version=3, is_pipeline=True)
         return df
 
     return _refresh

data/pipeline_handler.py

@@ -10,7 +10,7 @@ class PipelineHandler:
 
     def __init__(self):
         self.pipeline_infos = {}
-        self.github_base_url = "https://raw.githubusercontent.com/illuin-tech/vidore-benchmark/vidore_v3_pipeline/results"
+        self.github_base_url = "https://raw.githubusercontent.com/illuin-tech/vidore-benchmark/vidore_v3_pipeline/results/metrics"
         self.available_datasets = []
         self.available_languages = ["overall"]  # Default languages available
         
@@ -23,7 +23,7 @@ class PipelineHandler:
 
     def get_pipeline_folders_from_github(self) -> List[str]:
         """Get list of pipeline folders from GitHub API."""
-        api_url = "https://api.github.com/repos/illuin-tech/vidore-benchmark/contents/results?ref=vidore_v3_pipeline"
+        api_url = "https://api.github.com/repos/illuin-tech/vidore-benchmark/contents/results/metrics?ref=vidore_v3_pipeline"
         
         try:
             response = requests.get(api_url, headers=self.headers)
@@ -39,7 +39,7 @@ class PipelineHandler:
     
     def get_dataset_files_from_github(self, pipeline_name: str) -> List[str]:
         """Get list of dataset JSON files for a specific pipeline from GitHub API."""
-        api_url = f"https://api.github.com/repos/illuin-tech/vidore-benchmark/contents/results/{pipeline_name}?ref=vidore_v3_pipeline"
+        api_url = f"https://api.github.com/repos/illuin-tech/vidore-benchmark/contents/results/metrics/{pipeline_name}?ref=vidore_v3_pipeline"
 
         try:
             response = requests.get(api_url, headers=self.headers)