new_benchmark

.github/workflows/restart.yml

@@ -1,19 +0,0 @@
-name: Scheduled Restart
-
-on:
-  schedule:
-    - cron: '0 */1 * * *' # every 6 hours
-  workflow_dispatch:
-
-jobs:
-  restart:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          date > restart.txt
-          git config user.name "github-actions"
-          git config user.email "github-actions@github.com"
-          git add restart.txt
-          git commit -m "Scheduled restart"
-          git push

CHANGELOG.md

@@ -1,3 +0,0 @@
-# 0.1.0
-
-- We identified that the insurance policy data used in the Vidore V2 leaderboard falls within a grey area of licensing. While the data was publicly accessible, its licensing terms were not sufficiently clear to ensure compliance with our data usage standards. We have since removed this benchmark data from HuggingFace and removed the concerned columns from the leaderboard.

README.md

@@ -7,8 +7,7 @@ sdk: gradio
 app_file: app.py
 pinned: true
 license: mit
-sdk_version: 6.3.0
-python_version: 3.12.11
+sdk_version: 4.37.1
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,79 +1,20 @@
-import re
-
 import gradio as gr
-import pandas as pd
-import plotly.express as px
-
-from app.utils import (
-    add_rank_and_format,
-    deprecated_get_refresh_function,
-    filter_models,
-    get_pipeline_refresh_function,
-    get_refresh_function,
-)
-from data.deprecated_model_handler import DeprecatedModelHandler
-from data.model_handler import ModelHandler
-from data.pipeline_handler import PipelineHandler
 
-METRICS = [
-    "ndcg_at_1",
-    "ndcg_at_5",
-    "ndcg_at_10",
-    "ndcg_at_100",
-    "recall_at_1",
-    "recall_at_5",
-    "recall_at_10",
-    "recall_at_100",
-]
+from app.utils import add_rank_and_format, filter_models, get_refresh_function
+from data.model_handler import ModelHandler
 
+METRICS = ["ndcg_at_5", "recall_at_1"]
 
 def main():
-    # Get new results
     model_handler = ModelHandler()
     initial_metric = "ndcg_at_5"
 
-    data_benchmark_1 = model_handler.render_df(initial_metric, benchmark_version=1)
-    data_benchmark_1 = add_rank_and_format(data_benchmark_1, benchmark_version=1)
-
-    data_benchmark_2 = model_handler.render_df(initial_metric, benchmark_version=2)
-    data_benchmark_2 = add_rank_and_format(data_benchmark_2, benchmark_version=2)
-
-    num_datasets_1 = len(data_benchmark_1.columns) - 3
-    num_scores_1 = len(data_benchmark_1) * num_datasets_1
-    num_models_1 = len(data_benchmark_1)
-
-    num_datasets_2 = len(data_benchmark_2.columns) - 3
-    num_scores_2 = len(data_benchmark_2) * num_datasets_2
-    num_models_2 = len(data_benchmark_2)
-
-    deprecated_model_handler = DeprecatedModelHandler()
-    initial_metric = "ndcg_at_5"
-    initial_metric_v3 = "ndcg_at_10"
-
-    # Get pipeline evaluation results
-    pipeline_handler = PipelineHandler()
-    pipeline_handler.get_pipeline_data()
-    data_pipeline = pipeline_handler.render_df(initial_metric_v3, "english")
-    data_pipeline = add_rank_and_format(data_pipeline, benchmark_version=3, is_pipeline=True)
-
-    num_datasets_pipeline = len(data_pipeline.columns) - 5  # Excluding Rank, Model, Indexing time, search time, Average
-    num_scores_pipeline = len(data_pipeline) * num_datasets_pipeline
-    num_pipelines = len(data_pipeline)
-
-    deprecated_model_handler.get_vidore_data(initial_metric)
-    deprecated_data_benchmark_1 = deprecated_model_handler.render_df(initial_metric, benchmark_version=1)
-    deprecated_data_benchmark_1 = add_rank_and_format(deprecated_data_benchmark_1, benchmark_version=1)
-
-    deprecated_data_benchmark_2 = deprecated_model_handler.render_df(initial_metric, benchmark_version=2)
-    deprecated_data_benchmark_2 = add_rank_and_format(deprecated_data_benchmark_2, benchmark_version=2)
-
-    deprecated_num_datasets_1 = len(deprecated_data_benchmark_1.columns) - 3
-    deprecated_num_scores_1 = len(deprecated_data_benchmark_1) * deprecated_num_datasets_1
-    deprecated_num_models_1 = len(deprecated_data_benchmark_1)
+    data = model_handler.get_vidore_data(initial_metric)
+    data = add_rank_and_format(data)
 
-    deprecated_num_datasets_2 = len(deprecated_data_benchmark_2.columns) - 3
-    deprecated_num_scores_2 = len(deprecated_data_benchmark_2) * deprecated_num_datasets_2
-    deprecated_num_models_2 = len(deprecated_data_benchmark_2)
+    NUM_DATASETS = len(data.columns) - 3
+    NUM_SCORES = len(data) * NUM_DATASETS
+    NUM_MODELS = len(data)
 
     css = """
     table > thead {
@@ -96,812 +37,131 @@ def main():
     .markdown {
         font-size: 1rem;
     }
-
-    .alert-info {
-        background-color: #e3f2fd;
-        border-left: 4px solid #2196f3;
-        padding: 5px 15px;
-    }
-
-    /* 1. Force text wrapping on all headers from the 3rd column onwards */
-    #pipeline-table table th:nth-child(n+3) * {
-        white-space: normal !important;
-        overflow: visible !important;
-        text-overflow: clip !important;
-        line-height: 1.2 !important;
-        word-break: normal !important; /* Prevents breaking in the middle of words */
-        overflow-wrap: normal !important; /* Prevents breaking in the middle of words */
-    }
-
-    /* 2. Set a fixed width and center alignment for headers and data cells from the 3rd column onwards */
-    #pipeline-table table th:nth-child(n+3),
-    #pipeline-table table td:nth-child(n+3) {
-        min-width: 100px !important; /* Just wide enough to fit long words like "Pharmaceuticals" */
-        max-width: 120px !important;
-    }
-
-    /* 3. Make the Model column (2nd column) wider for pipeline table */
-    #pipeline-table table th:nth-child(2),
-    #pipeline-table table td:nth-child(2) {
-        min-width: 400px !important;
-        max-width: 500px !important;
-    }
     """
 
     with gr.Blocks(css=css) as block:
-        with gr.Tabs() as tabs:
-            with gr.TabItem("ViDoRe V3", id="vidore-v3"):
-                gr.Markdown("# ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-case 👷‍♂️")
-
-                gr.Markdown(
-                    """
-                Visual Document Retrieval Benchmark 3 leaderboard. To submit results, refer to the corresponding tab.
-
-                Refer to:
-                - 🤗 The [blogpost](https://huggingface.co/blog/QuentinJG/introducing-vidore-v3) for all the details on the datasets,
-                - 🤗 The [dataset collection](https://huggingface.co/collections/vidore/vidore-benchmark-v3),
-                - 📝 The [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics.
-                """
-                )
-
-                gr.Markdown("""
-                As the reference results are now hosted on the [MTEB Leaderboard](https://mteb-leaderboard.hf.space/?benchmark_name=ViDoRe(v3)),
-                we embed it here.
-                            """)
-
-                gr.Markdown(
-                    """**💡 To display English-only results:**
-                    - Under *Customize this Benchmark*, unselect the French datasets (*Vidore3EnergyRetrieval*, *Vidore3FinanceFrRetrieval*, *Vidore3PhysicsRetrieval*),
-                    - Go to the *Performance per language* tab (you might have to click on the three dots on the right of the tab bar to see it),
-                    - The *eng-Latn* column will show English-only results (= English queries on English documents).""",
-                    elem_classes="alert-info",
-                )
-
-                gr.HTML(
-                    """
-                <iframe
-                    src="https://mteb-leaderboard.hf.space/?benchmark_name=ViDoRe(v3)"
-                    style="width:100%; height:1000px; border:2px solid black; border-radius:10px;"
-                ></iframe>
-                """
-                )
-
-            with gr.TabItem("ViDoRe V3 (Pipeline)", id="vidore-v3-pipeline"):
-                gr.Markdown("# ViDoRe V3 (Pipeline Evaluation): Retrieval Performance for Complex Pipelines ⚙️")
-                gr.Markdown(
-                    "### Assessing retrieval performance, latency, and compute costs of complex retrieval pipelines"
-                )
-
-                gr.Markdown(
-                    """
-                    This leaderboard ranks full retrieval pipelines on **English-only queries** for **ViDoRe V3**. Instead of just testing standalone models, we evaluate real-world, multi-step retrieval systems. This includes everything from basic retrievers to advanced setups using AI agents, query reformulation, hybrid search, and any other creative retrieval pipeline one can imagine.
-
-                    To show efficiency alongside accuracy, we include **Indexing latency** (seconds/doc) and **Search latency** (seconds/query). These numbers are self-reported and depend heavily on hardware, so treat them as rough estimates. Still, they give a helpful idea of how much computing power each pipeline needs.
-
-                    ⚠️ **Note:** Because this only uses English queries, these scores **cannot be directly compared** to the standard ViDoRe V3 results.
-
-                    *Results are sourced from the [vidore-benchmark repository](https://github.com/illuin-tech/vidore-benchmark/tree/main/results).*
-                    """
-                )
-
-                if len(data_pipeline) > 0:
-                    datasets_columns_pipeline = [
-                        col
-                        for col in data_pipeline.columns[4:]
-                        if col not in ["Indexing latency (s/doc)", "Search latency (s/query)", "Average Score"]
-                    ]
-
-                    with gr.Row():
-                        metric_dropdown_pipeline = gr.Dropdown(
-                            choices=METRICS, value=initial_metric_v3, label="Select Metric"
-                        )
-                        research_textbox_pipeline = gr.Textbox(
-                            placeholder="🔍 Search Pipelines... [press enter]",
-                            label="Filter Pipelines by Name",
-                        )
-
-                    with gr.Row():
-                        if datasets_columns_pipeline:
-                            column_checkboxes_pipeline = gr.CheckboxGroup(
-                                choices=datasets_columns_pipeline,
-                                value=datasets_columns_pipeline,
-                                label="Select Datasets to Display",
-                            )
-                        else:
-                            column_checkboxes_pipeline = gr.CheckboxGroup(
-                                choices=[], value=[], label="Select Datasets to Display"
-                            )
-
-                    with gr.Row():
-                        # Datatype: Rank, Pipeline, Indexing latency (s/doc), Search latency (s/query), Average + datasets
-                        datatype_pipeline = ["number", "markdown", "number", "number", "number"] + ["number"] * len(
-                            datasets_columns_pipeline
-                        )
-                        dataframe_pipeline = gr.Dataframe(
-                            data_pipeline, datatype=datatype_pipeline, type="pandas", elem_id="pipeline-table"
-                        )
-
-                    def clean_pipeline_name(name):
-                        if not isinstance(name, str):
-                            return str(name)
-                        # Remove Markdown links [text](url) -> text
-                        name = re.sub(r"\[([^\]]+)\]\([^\)]+\)", r"\1", name)
-                        # Remove HTML tags <a href="...">text</a> -> text
-                        name = re.sub(r"<[^>]+>", "", name)
-                        return name.strip()
-
-                    def create_pipeline_plot(df, latency_col):
-                        if df is None or len(df) == 0:
-                            return None
-
-                        # Ensure expected columns exist
-                        if (
-                            latency_col not in df.columns
-                            or "Average Score" not in df.columns
-                            or "Pipeline" not in df.columns
-                        ):
-                            return None
-
-                        # Clean the dataframe for plotting
-                        plot_df = df.copy()
-
-                        # Strip HTML and Markdown for clean hover text
-                        plot_df["Cleaned Pipeline"] = plot_df["Pipeline"].apply(clean_pipeline_name)
-
-                        plot_df[latency_col] = pd.to_numeric(plot_df[latency_col], errors="coerce")
-                        plot_df["Average Score"] = pd.to_numeric(plot_df["Average Score"], errors="coerce")
-
-                        plot_df = plot_df.dropna(subset=[latency_col, "Average Score"])
-                        plot_df = plot_df[plot_df[latency_col] > 0]
-                        plot_df = plot_df.sort_values(by=latency_col)
-
-                        if len(plot_df) == 0:
-                            return None
-
-                        fig = px.scatter(
-                            plot_df,
-                            x=latency_col,
-                            y="Average Score",
-                            hover_name="Cleaned Pipeline",  # Use the clean text!
-                            title=f"Mean Performance vs {latency_col}",
-                            color_discrete_sequence=["orange"],
-                        )
-
-                        fig.update_layout(
-                            xaxis_title=latency_col,
-                            yaxis_title="Average Score",
-                            plot_bgcolor="white",
-                        )
-
-                        fig.update_xaxes(showgrid=True, gridcolor="lightgrey")
-                        fig.update_yaxes(showgrid=True, gridcolor="lightgrey")
-
-                        fig.update_traces(marker=dict(size=12, opacity=0.8, line=dict(width=1, color="DarkSlateGrey")))
-                        return fig
-
-                    with gr.Row():
-                        latency_radio = gr.Radio(
-                            choices=["Search latency (s/query)", "Indexing latency (s/doc)"],
-                            value="Search latency (s/query)",
-                            label="Select Latency Metric for X-Axis",
-                        )
-
-                    with gr.Row():
-                        initial_fig = create_pipeline_plot(data_pipeline, "Search latency (s/query)")
-                        performance_plot = gr.Plot(value=initial_fig)
-
-                    def update_data_pipeline(metric, search_term, selected_columns):
-                        pipeline_handler.get_pipeline_data()
-                        data = pipeline_handler.render_df(metric, "english")
-                        data = add_rank_and_format(
-                            data, benchmark_version=3, selected_columns=selected_columns, is_pipeline=True
-                        )
-                        data = filter_models(data, search_term)
-                        if selected_columns:
-                            # Include core columns plus selected dataset columns
-                            core_cols = ["Rank", "Pipeline", "Indexing latency (s/doc)", "Search latency (s/query)"]
-                            if "Average Score" in data.columns:
-                                core_cols.insert(4, "Average Score")
-                            data = data[core_cols + selected_columns]
-                        return data
-
-                    with gr.Row():
-                        refresh_button_pipeline = gr.Button("Refresh")
-                        refresh_button_pipeline.click(
-                            lambda metric: add_rank_and_format(
-                                pipeline_handler.render_df(metric, "english"), benchmark_version=3, is_pipeline=True
-                            ),
-                            inputs=[metric_dropdown_pipeline],
-                            outputs=dataframe_pipeline,
-                            concurrency_limit=20,
-                        ).then(
-                            fn=create_pipeline_plot,
-                            inputs=[dataframe_pipeline, latency_radio],
-                            outputs=performance_plot,
-                        )
-
-                    with gr.Row():
-                        gr.Markdown(
-                            """
-                        **Note**: These results represent full pipeline evaluations on english queries ONLY (since other queries were mostly directly translated from their english counterparts).
-                        We felt mutlilingual results were less critical (and much more costly to evaluate on the full set) for pipelines, since one user could just add a translation module to the pipeline and expect similar performance to the english results.
-                        If you feel this is a mistake and multi-lingual results are critical for pipelines, please let us know by opening an issue in the GitHub repository!
-                        """
-                        )
-
-                    # Automatically refresh the dataframe when the dropdown value changes
-                    def refresh_pipeline_data(metric):
-                        """Refresh pipeline data when metric changes."""
-                        df = pipeline_handler.render_df(metric, "english")
-                        return add_rank_and_format(df, benchmark_version=3, is_pipeline=True)
-
-                    # Update dataframe and then update the plot
-                    metric_dropdown_pipeline.change(
-                        refresh_pipeline_data,
-                        inputs=[metric_dropdown_pipeline],
-                        outputs=dataframe_pipeline,
-                    ).then(
-                        fn=create_pipeline_plot, inputs=[dataframe_pipeline, latency_radio], outputs=performance_plot
-                    )
-
-                    research_textbox_pipeline.submit(
-                        lambda metric, search_term, selected_columns: update_data_pipeline(
-                            metric, search_term, selected_columns
-                        ),
-                        inputs=[metric_dropdown_pipeline, research_textbox_pipeline, column_checkboxes_pipeline],
-                        outputs=dataframe_pipeline,
-                    ).then(
-                        fn=create_pipeline_plot, inputs=[dataframe_pipeline, latency_radio], outputs=performance_plot
-                    )
-
-                    column_checkboxes_pipeline.change(
-                        lambda metric, search_term, selected_columns: update_data_pipeline(
-                            metric, search_term, selected_columns
-                        ),
-                        inputs=[metric_dropdown_pipeline, research_textbox_pipeline, column_checkboxes_pipeline],
-                        outputs=dataframe_pipeline,
-                    ).then(
-                        fn=create_pipeline_plot, inputs=[dataframe_pipeline, latency_radio], outputs=performance_plot
-                    )
-
-                    # Update plot when the radio button changes
-                    latency_radio.change(
-                        fn=create_pipeline_plot, inputs=[dataframe_pipeline, latency_radio], outputs=performance_plot
-                    )
-
-                    gr.Markdown(
-                        f"""
-                    - **Total Datasets**: {num_datasets_pipeline}
-                    - **Total Scores**: {num_scores_pipeline}
-                    - **Total Pipelines**: {num_pipelines}
-                    """
-                        + r"""
-                    Please consider citing:
-
-                    ```bibtex
-                    @misc{faysse2024colpaliefficientdocumentretrieval,
-                      title={ColPali: Efficient Document Retrieval with Vision Language Models},
-                      author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
-                      year={2024},
-                      eprint={2407.01449},
-                      archivePrefix={arXiv},
-                      primaryClass={cs.IR},
-                      url={[https://arxiv.org/abs/2407.01449](https://arxiv.org/abs/2407.01449)},
-                    }
-                    @misc{loison2026vidore,
-                      title={ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
-                      author={Loison, Ant{\'o}nio and Mac{\'e}, Quentin and Edy, Antoine and Xing, Victor and Balough, Tom and Moreira, Gabriel and Liu, Bo and Faysse, Manuel and Hudelot, C{\'e}line and Viaud, Gautier},
-                      journal={arXiv preprint arXiv:2601.08620},
-                      year={2026}
-                    }
-                    ```
-                    """
-                    )
-                else:
-                    gr.Markdown("**No pipeline evaluation results available yet. Check back later!**")
-
-            with gr.TabItem("ViDoRe V2", id="vidore-v2"):
-                gr.Markdown("# ViDoRe V2: A new visual Document Retrieval Benchmark 📚🔍")
-                gr.Markdown("### A harder dataset benchmark for visual document retrieval 👀")
+        with gr.Tabs():
+            with gr.TabItem("🏆 Leaderboard"):
+                gr.Markdown("# ViDoRe: The Visual Document Retrieval Benchmark 📚🔍")
+                gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
 
                 gr.Markdown(
                     """
-                Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.
+                Visual Document Retrieval Benchmark leaderboard. To submit results, refer to the corresponding tab.  
 
-                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
+                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
                 """
                 )
-                datasets_columns_2 = list(data_benchmark_2.columns[4:])
+                datasets_columns = list(data.columns[3:])
+                anchor_columns = list(data.columns[:3])
+                default_columns = anchor_columns + datasets_columns
 
                 with gr.Row():
-                    metric_dropdown_2 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
-                    research_textbox_2 = gr.Textbox(
-                        placeholder="🔍 Search Models... [press enter]",
-                        label="Filter Models by Name",
-                    )
-                    column_checkboxes_2 = gr.CheckboxGroup(
-                        choices=datasets_columns_2, value=datasets_columns_2, label="Select Columns to Display"
-                    )
+                    metric_dropdown = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
+                    research_textbox = gr.Textbox(placeholder="🔍 Search Models... [press enter]", label="Filter Models by Name", )
+                    column_checkboxes = gr.CheckboxGroup(choices=datasets_columns, value=default_columns, label="Select Columns to Display")
 
                 with gr.Row():
-                    datatype_2 = ["number", "markdown"] + ["number"] * (num_datasets_2 + 1)
-                    dataframe_2 = gr.Dataframe(data_benchmark_2, datatype=datatype_2, type="pandas")
+                    datatype = ["number", "markdown"] + ["number"] * (NUM_DATASETS + 1)
+                    dataframe = gr.Dataframe(data, datatype=datatype, type="pandas")
 
-                def update_data_2(metric, search_term, selected_columns):
-                    model_handler.get_vidore_data(metric)
-                    data = model_handler.render_df(metric, benchmark_version=2)
-                    data = add_rank_and_format(data, benchmark_version=2, selected_columns=selected_columns)
+                def update_data(metric, search_term, selected_columns):
+                    data = model_handler.get_vidore_data(metric)
+                    data = add_rank_and_format(data)
                     data = filter_models(data, search_term)
-                    # data = remove_duplicates(data)  # Add this line
                     if selected_columns:
-                        data = data[
-                            ["Rank", "Model", "Model Size (Million Parameters)", "Average Score"] + selected_columns
-                        ]
+                        selected_columns = selected_columns
+                        data = data[selected_columns]
                     return data
 
                 with gr.Row():
-                    refresh_button_2 = gr.Button("Refresh")
-                    refresh_button_2.click(
-                        get_refresh_function(model_handler, benchmark_version=2),
-                        inputs=[metric_dropdown_2],
-                        outputs=dataframe_2,
-                        concurrency_limit=20,
-                    )
+                    refresh_button = gr.Button("Refresh")
+                    refresh_button.click(get_refresh_function(), inputs=[metric_dropdown], outputs=dataframe, concurrency_limit=20)
 
-                with gr.Row():
-                    gr.Markdown(
-                        """
-                    **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side.
-                    Those numbers are not numbers obtained from the organisations that released those models.
-                    """
-                    )
 
                 # Automatically refresh the dataframe when the dropdown value changes
-                metric_dropdown_2.change(
-                    get_refresh_function(model_handler, benchmark_version=2),
-                    inputs=[metric_dropdown_2],
-                    outputs=dataframe_2,
-                )
-                research_textbox_2.submit(
-                    lambda metric, search_term, selected_columns: update_data_2(metric, search_term, selected_columns),
-                    inputs=[metric_dropdown_2, research_textbox_2, column_checkboxes_2],
-                    outputs=dataframe_2,
+                metric_dropdown.change(get_refresh_function(), inputs=[metric_dropdown], outputs=dataframe)
+                research_textbox.submit(
+                    lambda metric, search_term, selected_columns: update_data(metric, search_term, selected_columns), 
+                    inputs=[metric_dropdown, research_textbox, column_checkboxes], 
+                    outputs=dataframe
                 )
-                column_checkboxes_2.change(
-                    lambda metric, search_term, selected_columns: update_data_2(metric, search_term, selected_columns),
-                    inputs=[metric_dropdown_2, research_textbox_2, column_checkboxes_2],
-                    outputs=dataframe_2,
+                column_checkboxes.change(
+                    lambda metric, search_term, selected_columns: update_data(metric, search_term, selected_columns), 
+                    inputs=[metric_dropdown, research_textbox, column_checkboxes], 
+                    outputs=dataframe
                 )
 
-                gr.Markdown(
-                    f"""
-                - **Total Datasets**: {num_datasets_2}
-                - **Total Scores**: {num_scores_2}
-                - **Total Models**: {num_models_2}
-                """
-                    + r"""
-                Please consider citing:
-
-                ```bibtex
-                @misc{faysse2024colpaliefficientdocumentretrieval,
-                  title={ColPali: Efficient Document Retrieval with Vision Language Models},
-                  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
-                  year={2024},
-                  eprint={2407.01449},
-                  archivePrefix={arXiv},
-                  primaryClass={cs.IR},
-                  url={https://arxiv.org/abs/2407.01449},
-                }
+                #column_checkboxes.change(get_refresh_function(), inputs=[metric_dropdown, column_checkboxes], outputs=dataframe)
 
-                @misc{macé2025vidorebenchmarkv2raising,
-                      title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
-                      author={Quentin Macé and António Loison and Manuel Faysse},
-                      year={2025},
-                      eprint={2505.17166},
-                      archivePrefix={arXiv},
-                      primaryClass={cs.IR},
-                      url={https://arxiv.org/abs/2505.17166},
-                }
-                ```
-                """
-                )
-            with gr.TabItem("ViDoRe V1", id="vidore-v1"):
-                gr.Markdown("# ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
-                gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
-
-                gr.Markdown(
-                    """
-                Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.
-
-                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
-                """
-                )
-                datasets_columns_1 = list(data_benchmark_1.columns[4:])
-
-                with gr.Row():
-                    metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
-                    research_textbox_1 = gr.Textbox(
-                        placeholder="🔍 Search Models... [press enter]",
-                        label="Filter Models by Name",
-                    )
-                    column_checkboxes_1 = gr.CheckboxGroup(
-                        choices=datasets_columns_1, value=datasets_columns_1, label="Select Columns to Display"
-                    )
-
-                with gr.Row():
-                    datatype_1 = ["number", "markdown"] + ["number"] * (num_datasets_1 + 1)
-                    dataframe_1 = gr.Dataframe(data_benchmark_1, datatype=datatype_1, type="pandas")
-
-                def update_data_1(metric, search_term, selected_columns):
-                    model_handler.get_vidore_data(metric)
-                    data = model_handler.render_df(metric, benchmark_version=1)
-                    data = add_rank_and_format(data, benchmark_version=1, selected_columns=selected_columns)
-                    data = filter_models(data, search_term)
-                    if selected_columns:
-                        data = data[
-                            ["Rank", "Model", "Model Size (Million Parameters)", "Average Score"] + selected_columns
-                        ]
-                    return data
-
-                with gr.Row():
-                    refresh_button_1 = gr.Button("Refresh")
-                    refresh_button_1.click(
-                        get_refresh_function(model_handler, benchmark_version=1),
-                        inputs=[metric_dropdown_1],
-                        outputs=dataframe_1,
-                        concurrency_limit=20,
-                    )
-
-                # Automatically refresh the dataframe when the dropdown value changes
-                metric_dropdown_1.change(
-                    get_refresh_function(model_handler, benchmark_version=1),
-                    inputs=[metric_dropdown_1],
-                    outputs=dataframe_1,
-                )
-                research_textbox_1.submit(
-                    lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
-                    inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
-                    outputs=dataframe_1,
-                )
-                column_checkboxes_1.change(
-                    lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
-                    inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
-                    outputs=dataframe_1,
-                )
 
                 gr.Markdown(
                     f"""
-                - **Total Datasets**: {num_datasets_1}
-                - **Total Scores**: {num_scores_1}
-                - **Total Models**: {num_models_1}
+                - **Total Datasets**: {NUM_DATASETS}
+                - **Total Scores**: {NUM_SCORES}
+                - **Total Models**: {NUM_MODELS}
                 """
                     + r"""
                 Please consider citing:
 
                 ```bibtex
                 @misc{faysse2024colpaliefficientdocumentretrieval,
-                  title={ColPali: Efficient Document Retrieval with Vision Language Models},
+                  title={ColPali: Efficient Document Retrieval with Vision Language Models}, 
                   author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
                   year={2024},
                   eprint={2407.01449},
                   archivePrefix={arXiv},
                   primaryClass={cs.IR},
-                  url={https://arxiv.org/abs/2407.01449},
+                  url={https://arxiv.org/abs/2407.01449}, 
                 }
-                @misc{macé2025vidorebenchmarkv2raising,
-                  title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
-                  author={Quentin Macé and António Loison and Manuel Faysse},
-                  year={2025},
-                  eprint={2505.17166},
-                  archivePrefix={arXiv},
-                  primaryClass={cs.IR},
-                  url={https://arxiv.org/abs/2505.17166},
-                }
-
                 ```
                 """
                 )
-            with gr.TabItem("📚 Submit your model", id="submission"):
+            with gr.TabItem("📚 Submit your model"):
                 gr.Markdown("# How to Submit a New Model to the Leaderboard")
                 gr.Markdown(
                     """
-                    ## Original (ViDoRe v1-v3) leaderboard:
-                    To submit a new model to the original ViDoRe leaderboard, follow these steps:
+                    To submit a new model to the ViDoRe leaderboard, follow these steps:
 
                     1. **Evaluate your model**:
-                       - Follow the evaluation procedure provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/) that uses MTEB.
+                       - Follow the evaluation script provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/)
 
                     2. **Format your submission file**:
-                        - Add the generated files to [MTEB results](https://github.com/embeddings-benchmark/results) project. Check the [Colpali results](https://github.com/embeddings-benchmark/results/tree/main/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0) for an example.
+                        - The submission file should automatically be generated, and named `results.json` with the
+                        following structure:
+                        ```json
+                        {
+                            "dataset_name_1": {
+                                "metric_1": score_1,
+                                "metric_2": score_2,
+                                ...
+                            },
+                            "dataset_name_2": {
+                                "metric_1": score_1,
+                                "metric_2": score_2,
+                                ...
+                            },
+                        }
+                        ```
+                        - The dataset names should be the same as the ViDoRe dataset names listed in the following
+                        collection: [ViDoRe Benchmark](https://huggingface.co/collections/vidore/vidore-benchmark-667173f98e70a1c0fa4db00d).
+
+                    3. **Submit your model**:
+                        - Create a public HuggingFace model repository with your model.
+                        - Add the tag `vidore` to your model in the metadata of the model card and place the
+                        `results.json` file at the root.
 
                     And you're done! Your model will appear on the leaderboard when you click refresh! Once the space
                     gets rebooted, it will appear on startup.
 
                     Note: For proper hyperlink redirection, please ensure that your model repository name is in
                     kebab-case, e.g. `my-model-name`.
-
-
-                    ## ViDoRe v3 Pipeline leaderboard :
-                    To submit a new pipeline to the ViDoRe V3 pipeline leaderboard, follow these steps:
-                    1. **Evaluate your pipeline**:
-                       - Follow the evaluation procedure provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/) for pipelines
-                    2. **Open a PR on the ViDoRe GitHub repository including**:
-                        - Your results, which are directly outputed in the correct format
-                        - A short description of the pipeline and the main components used (some examples are available in the repository)
-
-                    And you're done! Your pipeline will appear on the leaderboard when after the PR is merged !
-                    """
-                )
-            with gr.TabItem("Deprecated ViDoRe V1", id="vidore-v1-deprecated"):
-                gr.Markdown(
-                    "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
-                    "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
-                    "which is no longer maintained. Results should be computed using the "
-                    "[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
-                    "[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
-                )
-                gr.Markdown(
-                    "## <span style='color:red'>Missing results in the new leaderboard are being added as they are re-computed.</span>"
-                )
-                gr.Markdown(
-                    "# <span style='color:red'>[Deprecated]</span> ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍"
-                )
-                gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
-
-                gr.Markdown(
-                    """
-                Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.
-
-                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
-                """
-                )
-                deprecated_datasets_columns_1 = list(deprecated_data_benchmark_1.columns[3:])
-
-                with gr.Row():
-                    deprecated_metric_dropdown_1 = gr.Dropdown(
-                        choices=METRICS, value=initial_metric, label="Select Metric"
-                    )
-                    deprecated_research_textbox_1 = gr.Textbox(
-                        placeholder="🔍 Search Models... [press enter]",
-                        label="Filter Models by Name",
-                    )
-                    deprecated_column_checkboxes_1 = gr.CheckboxGroup(
-                        choices=deprecated_datasets_columns_1,
-                        value=deprecated_datasets_columns_1,
-                        label="Select Columns to Display",
-                    )
-
-                with gr.Row():
-                    deprecated_datatype_1 = ["number", "markdown"] + ["number"] * (deprecated_num_datasets_1 + 1)
-                    deprecated_dataframe_1 = gr.Dataframe(
-                        deprecated_data_benchmark_1, datatype=deprecated_datatype_1, type="pandas"
-                    )
-
-                def deprecated_update_data_1(metric, search_term, selected_columns):
-                    deprecated_model_handler.get_vidore_data(metric)
-                    data = deprecated_model_handler.render_df(metric, benchmark_version=1)
-                    data = add_rank_and_format(data, benchmark_version=1, selected_columns=selected_columns)
-                    data = filter_models(data, search_term)
-                    # data = remove_duplicates(data)  # Add this line
-                    if selected_columns:
-                        data = data[["Rank", "Model", "Average Score"] + selected_columns]
-                    return data
-
-                with gr.Row():
-                    deprecated_refresh_button_1 = gr.Button("Refresh")
-                    deprecated_refresh_button_1.click(
-                        deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=1),
-                        inputs=[deprecated_metric_dropdown_1],
-                        outputs=deprecated_dataframe_1,
-                        concurrency_limit=20,
-                    )
-
-                # Automatically refresh the dataframe when the dropdown value changes
-                deprecated_metric_dropdown_1.change(
-                    deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=1),
-                    inputs=[deprecated_metric_dropdown_1],
-                    outputs=deprecated_dataframe_1,
-                )
-                deprecated_research_textbox_1.submit(
-                    lambda metric, search_term, selected_columns: deprecated_update_data_1(
-                        metric, search_term, selected_columns
-                    ),
-                    inputs=[
-                        deprecated_metric_dropdown_1,
-                        deprecated_research_textbox_1,
-                        deprecated_column_checkboxes_1,
-                    ],
-                    outputs=deprecated_dataframe_1,
-                )
-                deprecated_column_checkboxes_1.change(
-                    lambda metric, search_term, selected_columns: deprecated_update_data_1(
-                        metric, search_term, selected_columns
-                    ),
-                    inputs=[
-                        deprecated_metric_dropdown_1,
-                        deprecated_research_textbox_1,
-                        deprecated_column_checkboxes_1,
-                    ],
-                    outputs=deprecated_dataframe_1,
-                )
-
-                gr.Markdown(
-                    f"""
-                - **Total Datasets**: {deprecated_num_datasets_1}
-                - **Total Scores**: {deprecated_num_scores_1}
-                - **Total Models**: {deprecated_num_models_1}
-                """
-                    + r"""
-                Please consider citing:
-
-                ```bibtex
-                @misc{faysse2024colpaliefficientdocumentretrieval,
-                  title={ColPali: Efficient Document Retrieval with Vision Language Models},
-                  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
-                  year={2024},
-                  eprint={2407.01449},
-                  archivePrefix={arXiv},
-                  primaryClass={cs.IR},
-                  url={https://arxiv.org/abs/2407.01449},
-                }
-
-                @misc{macé2025vidorebenchmarkv2raising,
-                  title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
-                  author={Quentin Macé and António Loison and Manuel Faysse},
-                  year={2025},
-                  eprint={2505.17166},
-                  archivePrefix={arXiv},
-                  primaryClass={cs.IR},
-                  url={https://arxiv.org/abs/2505.17166},
-                }
-                ```
-                """
-                )
-            with gr.TabItem("Deprecated ViDoRe V2", id="vidore-v2-deprecated"):
-                gr.Markdown(
-                    "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
-                    "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
-                    "which is no longer maintained. Results should be computed using the "
-                    "[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
-                    "[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
-                )
-                gr.Markdown(
-                    "## <span style='color:red'>Missing results in the new leaderboard are being added as they are re-computed.</span>"
-                )
-                gr.Markdown(
-                    "# <span style='color:red'>[Deprecated]</span> ViDoRe V2: A new visual Document Retrieval Benchmark 📚🔍"
-                )
-                gr.Markdown("### A harder dataset benchmark for visual document retrieval 👀")
-
-                gr.Markdown(
                     """
-                Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.
-
-                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
-                """
                 )
-                deprecated_datasets_columns_2 = list(deprecated_data_benchmark_2.columns[3:])
-
-                with gr.Row():
-                    deprecated_metric_dropdown_2 = gr.Dropdown(
-                        choices=METRICS, value=initial_metric, label="Select Metric"
-                    )
-                    deprecated_research_textbox_2 = gr.Textbox(
-                        placeholder="🔍 Search Models... [press enter]",
-                        label="Filter Models by Name",
-                    )
-                    deprecated_column_checkboxes_2 = gr.CheckboxGroup(
-                        choices=deprecated_datasets_columns_2,
-                        value=deprecated_datasets_columns_2,
-                        label="Select Columns to Display",
-                    )
 
-                with gr.Row():
-                    deprecated_datatype_2 = ["number", "markdown"] + ["number"] * (deprecated_num_datasets_2 + 1)
-                    deprecated_dataframe_2 = gr.Dataframe(
-                        deprecated_data_benchmark_2, datatype=deprecated_datatype_2, type="pandas"
-                    )
-
-                def deprecated_update_data_2(metric, search_term, selected_columns):
-                    deprecated_model_handler.get_vidore_data(metric)
-                    data = deprecated_model_handler.render_df(metric, benchmark_version=2)
-                    data = add_rank_and_format(data, benchmark_version=2, selected_columns=selected_columns)
-                    data = filter_models(data, search_term)
-                    # data = remove_duplicates(data)  # Add this line
-                    if selected_columns:
-                        data = data[["Rank", "Model", "Average Score"] + selected_columns]
-                    return data
-
-                with gr.Row():
-                    deprecated_refresh_button_2 = gr.Button("Refresh")
-                    deprecated_refresh_button_2.click(
-                        deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=2),
-                        inputs=[deprecated_metric_dropdown_2],
-                        outputs=deprecated_dataframe_2,
-                        concurrency_limit=20,
-                    )
-
-                with gr.Row():
-                    gr.Markdown(
-                        """
-                    **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side.
-                    Those numbers are not numbers obtained from the organisations that released those models.
-                    """
-                    )
-
-                # Automatically refresh the dataframe when the dropdown value changes
-                deprecated_metric_dropdown_2.change(
-                    deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=2),
-                    inputs=[deprecated_metric_dropdown_2],
-                    outputs=deprecated_dataframe_2,
-                )
-                deprecated_research_textbox_2.submit(
-                    lambda metric, search_term, selected_columns: deprecated_update_data_2(
-                        metric, search_term, selected_columns
-                    ),
-                    inputs=[
-                        deprecated_metric_dropdown_2,
-                        deprecated_research_textbox_2,
-                        deprecated_column_checkboxes_2,
-                    ],
-                    outputs=deprecated_dataframe_2,
-                )
-                deprecated_column_checkboxes_2.change(
-                    lambda metric, search_term, selected_columns: deprecated_update_data_2(
-                        metric, search_term, selected_columns
-                    ),
-                    inputs=[
-                        deprecated_metric_dropdown_2,
-                        deprecated_research_textbox_2,
-                        deprecated_column_checkboxes_2,
-                    ],
-                    outputs=deprecated_dataframe_2,
-                )
-
-                gr.Markdown(
-                    f"""
-                - **Total Datasets**: {deprecated_num_datasets_2}
-                - **Total Scores**: {deprecated_num_scores_2}
-                - **Total Models**: {deprecated_num_models_2}
-                """
-                    + r"""
-                Please consider citing:
-
-                ```bibtex
-                @misc{faysse2024colpaliefficientdocumentretrieval,
-                  title={ColPali: Efficient Document Retrieval with Vision Language Models},
-                  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
-                  year={2024},
-                  eprint={2407.01449},
-                  archivePrefix={arXiv},
-                  primaryClass={cs.IR},
-                  url={https://arxiv.org/abs/2407.01449},
-                }
-
-                @misc{macé2025vidorebenchmarkv2raising,
-                      title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
-                      author={Quentin Macé and António Loison and Manuel Faysse},
-                      year={2025},
-                      eprint={2505.17166},
-                      archivePrefix={arXiv},
-                      primaryClass={cs.IR},
-                      url={https://arxiv.org/abs/2505.17166},
-                }
-                ```
-                """
-                )
-
-        def select_tab_from_url(request: gr.Request):
-            # Grab query parameters from the URL
-            query_params = dict(request.query_params)
-            # Look for ?tab=..., default to the first tab's ID if not found
-            target_tab = query_params.get("tab", "vidore-v3")
-            # Update the tabs component to select the target ID
-            return gr.update(selected=target_tab)
-
-        block.load(select_tab_from_url, inputs=None, outputs=tabs)
     block.queue(max_size=10).launch(debug=True)
 
 
-if __name__ == "__main__":
+if __name__ == "__main__":    
     main()
+

app/utils.py

@@ -1,143 +1,31 @@
-import pandas as pd
-import math
+from data.model_handler import ModelHandler
 
+def make_clickable_model(model_name, link=None):
+    if link is None:
+        desanitized_model_name = model_name.replace("_", "/")
 
-def make_clickable_model(model_name, link=None, is_pipeline=False, folder_name=None):
-    if is_pipeline:
-        # For pipelines: use folder_name for link, model_name (alias) for display
-        link_folder = folder_name if folder_name else model_name
-        # Process folder name for link: only handle __ and -thisisapoint-
-        desanitized_folder = link_folder.replace("__", "/")
-        desanitized_folder = desanitized_folder.replace("-thisisapoint-", ".")
-        if link is None:
-            link = f"https://github.com/illuin-tech/vidore-benchmark/tree/main/results/pipeline_descriptions/{desanitized_folder}/description.json"
-        # Use word-wrap styling for potentially long pipeline aliases
-        return f'<a target="_blank" style="text-decoration: underline; word-wrap: break-word; white-space: normal; display: inline-block; max-width: 450px;" href="{link}">{model_name}</a>'
-    else:
-        # For regular models: replace __ and _ with /, and -thisisapoint- with .
-        desanitized_model_name = model_name.replace("__", "/")
-        desanitized_model_name = desanitized_model_name.replace("_", "/")
-        desanitized_model_name = desanitized_model_name.replace("-thisisapoint-", ".")
+        if '/captioning' in desanitized_model_name:
+            desanitized_model_name = desanitized_model_name.replace('/captioning', '')
+        if '/ocr' in desanitized_model_name:
+            desanitized_model_name = desanitized_model_name.replace('/ocr', '')
 
-        if "/captioning" in desanitized_model_name:
-            desanitized_model_name = desanitized_model_name.replace("/captioning", "")
-        if "/ocr" in desanitized_model_name:
-            desanitized_model_name = desanitized_model_name.replace("/ocr", "")
+        link = "https://huggingface.co/" + desanitized_model_name
 
-        if link is None:
-            link = "https://huggingface.co/" + desanitized_model_name
+    return f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name}</a>'
 
-    return f'<a target="_blank" style="text-decoration: underline" href="{link}">{desanitized_model_name}</a>'
 
-
-def add_rank(df, benchmark_version=1, selected_columns=None):
-    # Convert numeric columns to proper float type (they may be 'object' dtype due to mixed data)
-    for col in df.columns:
-        if col not in ["Model", "Pipeline", "_folder_name"]:
-            df[col] = pd.to_numeric(df[col], errors="coerce")
-
-    # Only fill NaN for numeric columns to avoid issues with string columns like _folder_name
-    numeric_cols = df.select_dtypes(include=["float64", "int64", "float32", "int32"]).columns
-    df[numeric_cols] = df[numeric_cols].fillna(0.0)
-
-    if selected_columns is None:
-        cols_to_rank = [
-            col
-            for col in df.columns
-            if col
-            not in [
-                "Model",
-                "Pipeline",  # For pipeline tables
-                "Model Size (Million Parameters)",
-                "Memory Usage (GB, fp32)",
-                "Embedding Dimensions",
-                "Max Tokens",
-                "Compute Cost ($)",
-                "Queries per Second",
-                "_folder_name",  # Hidden column for pipeline link generation
-            ]
-        ]
-    else:
-        cols_to_rank = selected_columns
-
-    if len(cols_to_rank) == 1:
-        df.sort_values(cols_to_rank[0], ascending=False, inplace=True)
-    else:
-        # Only add Average column if it doesn't already exist
-        if "Average Score" not in df.columns:
-            df.insert(len(df.columns) - len(cols_to_rank), "Average Score", df[cols_to_rank].mean(axis=1, skipna=False))
-        df.sort_values("Average Score", ascending=False, inplace=True)
-    df.insert(0, "Rank", list(range(1, len(df) + 1)))
-    # multiply values by 100 if they are floats and round to 2 decimal places
-    for col in df.columns:
-        if pd.api.types.is_numeric_dtype(df[col]) and col not in [
-            "Model Size (Million Parameters)",
-            "Compute Cost ($)",
-            "Queries per Second",
-            "Indexing latency (s/doc)",
-            "Search latency (s/query)",
-            "Rank",
-        ]:
-            df[col] = df[col].apply(lambda x: round(x * 100, 3 - int(math.floor(math.log10(abs(x * 100)))) - 1))
-        elif pd.api.types.is_numeric_dtype(df[col]) and col in ["Indexing latency (s/doc)", "Search latency (s/query)"]:
-            df[col] = df[col].apply(lambda x: round(x, 3 - int(math.floor(math.log10(abs(x)))) - 1))
-    return df
-
-
-def add_rank_and_format(df, benchmark_version=1, selected_columns=None, is_pipeline=False):
+def add_rank_and_format(df):
     df = df.reset_index()
-    column_name = "Pipeline" if is_pipeline else "Model"
-    df = df.rename(columns={"index": column_name})
-    df = add_rank(df, benchmark_version, selected_columns)
-
-    if is_pipeline and "_folder_name" in df.columns:
-        # For pipelines, use folder_name for link generation
-        df[column_name] = df.apply(
-            lambda row: make_clickable_model(row[column_name], is_pipeline=True, folder_name=row["_folder_name"]),
-            axis=1,
-        )
-        df = df.drop(columns=["_folder_name"])
-    else:
-        df[column_name] = df[column_name].apply(lambda x: make_clickable_model(x, is_pipeline=is_pipeline))
-    # df = remove_duplicates(df)
-    return df
-
-
-def remove_duplicates(df):
-    """Remove duplicate models based on their name (after the last '/' if present)."""
-    df["model_name"] = df["Model"].str.replace("_", "/")
-    df = df.sort_values("Rank").drop_duplicates(subset=["model_name"], keep="first")
-    df = df.drop("model_name", axis=1)
+    df = df.rename(columns={"index": "Model"})
+    df = ModelHandler.add_rank(df)
+    df["Model"] = df["Model"].apply(make_clickable_model)
     return df
 
-
-def get_refresh_function(model_handler, benchmark_version):
-    def _refresh(metric):
-        model_handler.get_vidore_data(metric)
-        data_task_category = model_handler.render_df(metric, benchmark_version)
-        df = add_rank_and_format(data_task_category, benchmark_version)
-        return df
-
-    return _refresh
-
-
-def deprecated_get_refresh_function(model_handler, benchmark_version):
-    def _refresh(metric):
-        model_handler.get_vidore_data(metric)
-        data_task_category = model_handler.render_df(metric, benchmark_version)
-        df = add_rank_and_format(data_task_category, benchmark_version)
-        return df
-
-    return _refresh
-
-
-def get_pipeline_refresh_function(pipeline_handler):
-    """Refresh function for pipeline evaluation results."""
-
+def get_refresh_function():
     def _refresh(metric):
-        pipeline_handler.get_pipeline_data()
-        data = pipeline_handler.render_df(metric)
-        df = add_rank_and_format(data, benchmark_version=3, is_pipeline=True)
+        model_handler = ModelHandler()
+        data_task_category = model_handler.get_vidore_data(metric)
+        df = add_rank_and_format(data_task_category)
         return df
 
     return _refresh
@@ -145,7 +33,5 @@ def get_pipeline_refresh_function(pipeline_handler):
 
 def filter_models(data, search_term):
     if search_term:
-        # Use "Pipeline" column for pipeline tables, "Model" for others
-        col_name = "Pipeline" if "Pipeline" in data.columns else "Model"
-        data = data[data[col_name].str.contains(search_term, case=False, na=False)]
-    return data
+        data = data[data['Model'].str.contains(search_term, case=False, na=False)]
+    return data

data/dataset_handler.py

@@ -1,22 +1,4 @@
-VIDORE_V1_MTEB_NAMES = [
-    "VidoreArxivQARetrieval",
-    "VidoreDocVQARetrieval",
-    "VidoreInfoVQARetrieval",
-    "VidoreShiftProjectRetrieval",
-    "VidoreSyntheticDocQAAIRetrieval",
-    "VidoreSyntheticDocQAEnergyRetrieval",
-    "VidoreSyntheticDocQAGovernmentReportsRetrieval",
-    "VidoreSyntheticDocQAHealthcareIndustryRetrieval",
-    "VidoreTabfquadRetrieval",
-    "VidoreTatdqaRetrieval",
-]
-VIDORE_V2_MTEB_NAMES = [
-    "Vidore2BioMedicalLecturesRetrieval",
-    "Vidore2EconomicsReportsRetrieval",
-    "Vidore2ESGReportsHLRetrieval",
-    "Vidore2ESGReportsRetrieval",
-]
-DEPRECATED_VIDORE_DATASETS_KEYWORDS = [
+VIDORE_DATASETS_KEYWORDS = [
     "arxivqa",
     "docvqa",
     "infovqa",
@@ -29,53 +11,8 @@ DEPRECATED_VIDORE_DATASETS_KEYWORDS = [
     "healthcare_industry",
 ]
 
-DEPRECATED_VIDORE_2_DATASETS_KEYWORDS = [
-    "restaurant_esg",
-    "rse_restaurant",
-    "mit_biomedical",
-    "economics_macro",
-    "biomedical_lectures",
-    "esg_reports",
-    "economics_reports",
-]
-
 
 def get_datasets_nickname(dataset_name) -> str:
-    if dataset_name == "VidoreArxivQARetrieval":
-        return "ArxivQA"
-    elif dataset_name == "VidoreDocVQARetrieval":
-        return "DocVQA"
-    elif dataset_name == "VidoreInfoVQARetrieval":
-        return "InfoVQA"
-    elif dataset_name == "VidoreTabfquadRetrieval":
-        return "TabFQuad"
-    elif dataset_name == "VidoreTatdqaRetrieval":
-        return "TAT-DQA"
-    elif dataset_name == "VidoreShiftProjectRetrieval":
-        return "Shift Project"
-    elif dataset_name == "VidoreSyntheticDocQAAIRetrieval":
-        return "Artificial Intelligence"
-    elif dataset_name == "VidoreSyntheticDocQAEnergyRetrieval":
-        return "Energy"
-    elif dataset_name == "VidoreSyntheticDocQAGovernmentReportsRetrieval":
-        return "Government Reports"
-    elif dataset_name == "VidoreSyntheticDocQAHealthcareIndustryRetrieval":
-        return "Healthcare Industry"
-
-    elif dataset_name == "Vidore2ESGReportsHLRetrieval":
-        return "ESG Restaurant Human English"
-    elif dataset_name == "Vidore2ESGReportsRetrieval":
-        return "ESG Restaurant Synthetic Multilingual"
-    elif dataset_name == "Vidore2BioMedicalLecturesRetrieval":
-        return "MIT Biomedical Multilingual"
-    elif dataset_name == "Vidore2EconomicsReportsRetrieval":
-        return "Economics Macro Multilingual"
-
-    else:
-        raise ValueError(f"Dataset {dataset_name} not found in ViDoRe")
-
-
-def deprecated_get_datasets_nickname(dataset_name) -> str:
     if "arxivqa" in dataset_name:
         return "ArxivQA"
 
@@ -104,32 +41,5 @@ def deprecated_get_datasets_nickname(dataset_name) -> str:
     elif "healthcare_industry" in dataset_name:
         return "Healthcare Industry"
 
-    elif ("restaurant_esg" in dataset_name) or ("esg_reports_human" in dataset_name):
-        return "ESG Restaurant Human"
-
-    elif ("rse_restaurant" in dataset_name and "multilingual" in dataset_name) or (
-        "esg_reports" in dataset_name and "_eng_" not in dataset_name
-    ):
-        return "ESG Restaurant Synthetic Multilingual"
-
-    elif ("rse_restaurant" in dataset_name) or ("esg_reports" in dataset_name and "_eng_" in dataset_name):
-        return "ESG Restaurant Synthetic"
-
-    elif ("mit_biomedical" in dataset_name and "multilingual" in dataset_name) or (
-        "biomedical_lectures" in dataset_name and "_eng_" not in dataset_name
-    ):
-        return "MIT Biomedical Multilingual"
-
-    elif ("mit_biomedical" in dataset_name) or ("biomedical_lectures" in dataset_name and "_eng_" in dataset_name):
-        return "MIT Biomedical"
-
-    elif ("economics_macro" in dataset_name and "multilingual" in dataset_name) or (
-        "economics_reports" in dataset_name and "_eng_" not in dataset_name
-    ):
-        return "Economics Macro Multilingual"
-
-    elif ("economics_macro" in dataset_name) or ("economics_reports" in dataset_name and "_eng_" in dataset_name):
-        return "Economics Macro"
-
     else:
         raise ValueError(f"Dataset {dataset_name} not found in ViDoRe")

data/deprecated_model_handler.py

@@ -1,128 +0,0 @@
-import json
-import os
-from typing import Any, Dict
-
-import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download, metadata_load
-
-from .dataset_handler import (
-    DEPRECATED_VIDORE_2_DATASETS_KEYWORDS,
-    DEPRECATED_VIDORE_DATASETS_KEYWORDS,
-    deprecated_get_datasets_nickname,
-)
-
-BLOCKLIST = ["impactframes"]
-
-
-class DeprecatedModelHandler:
-    def __init__(self, model_infos_path="model_infos.json"):
-        self.api = HfApi()
-        self.model_infos_path = model_infos_path
-        self.model_infos = self._load_model_infos()
-
-    def _load_model_infos(self) -> Dict:
-        if os.path.exists(self.model_infos_path):
-            with open(self.model_infos_path) as f:
-                return json.load(f)
-        return {}
-
-    def _save_model_infos(self):
-        with open(self.model_infos_path, "w") as f:
-            json.dump(self.model_infos, f)
-
-    def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
-        return "metadata" in results and "metrics" in results
-
-    def _is_baseline_repo(self, repo_id: str) -> bool:
-        return repo_id == "vidore/baseline-results"
-
-    def sanitize_model_name(self, model_name):
-        return model_name.replace("/", "_").replace(".", "-thisisapoint-")
-
-    def fuze_model_infos(self, model_name, results):
-        for dataset, metrics in results.items():
-            if dataset not in self.model_infos[model_name]["results"].keys():
-                self.model_infos[model_name]["results"][dataset] = metrics
-            else:
-                continue
-
-    def get_vidore_data(self, metric="ndcg_at_5"):
-        models = self.api.list_models(filter="vidore")
-        repositories = [model.modelId for model in models]  # type: ignore
-
-        # Sort repositories to process non-baseline repos first (to prioritize their results)
-        repositories.sort(key=lambda x: self._is_baseline_repo(x))
-
-        for repo_id in repositories:
-            org_name = repo_id.split("/")[0]
-            if org_name in BLOCKLIST:
-                continue
-            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith("_metrics.json") or f == "results.json"]
-
-            if len(files) == 0:
-                continue
-            else:
-                for file in files:
-                    if file.endswith("results.json"):
-                        model_name = repo_id.replace("/", "_").replace(".", "-thisisapoint-")
-                    else:
-                        model_name = file.split("_metrics.json")[0]
-                        model_name = model_name.replace("/", "_").replace(".", "-thisisapoint-")
-
-                    # Skip if the model is from baseline and we already have results
-
-                    readme_path = hf_hub_download(repo_id, filename="README.md")
-                    meta = metadata_load(readme_path)
-                    try:
-                        result_path = hf_hub_download(repo_id, filename=file)
-
-                        with open(result_path) as f:
-                            results = json.load(f)
-
-                        if self._are_results_in_new_vidore_format(results):
-                            metadata = results["metadata"]
-                            results = results["metrics"]
-
-                        # Handles the case where the model is both in baseline and outside of it
-                        # (prioritizes the non-baseline results)
-                        if self._is_baseline_repo(repo_id) and self.sanitize_model_name(model_name) in self.model_infos:
-                            self.fuze_model_infos(model_name, results)
-
-                        self.model_infos[model_name] = {"meta": meta, "results": results}
-                    except Exception as e:
-                        print(f"Error loading {model_name} - {e}")
-                        continue
-
-    # In order to keep only models relevant to a benchmark
-    def filter_models_by_benchmark(self, benchmark_version=1):
-        filtered_model_infos = {}
-        keywords = DEPRECATED_VIDORE_DATASETS_KEYWORDS if benchmark_version == 1 else DEPRECATED_VIDORE_2_DATASETS_KEYWORDS
-
-        for model, info in self.model_infos.items():
-            results = info["results"]
-            if any(any(keyword in dataset for keyword in keywords) for dataset in results.keys()):
-                filtered_model_infos[model] = info
-
-        return filtered_model_infos
-
-    # Compute the average of a metric for each model,
-    def render_df(self, metric="ndcg_at_5", benchmark_version=1):
-        model_res = {}
-        filtered_model_infos = self.filter_models_by_benchmark(benchmark_version)
-        if len(filtered_model_infos) > 0:
-            for model in filtered_model_infos.keys():
-                res = filtered_model_infos[model]["results"]
-                dataset_res = {}
-                keywords = DEPRECATED_VIDORE_DATASETS_KEYWORDS if benchmark_version == 1 else DEPRECATED_VIDORE_2_DATASETS_KEYWORDS
-                for dataset in res.keys():
-                    if not any(keyword in dataset for keyword in keywords):
-                        continue
-
-                    dataset_nickname = deprecated_get_datasets_nickname(dataset)
-                    dataset_res[dataset_nickname] = res[dataset][metric]
-                model_res[model] = dataset_res
-
-            df = pd.DataFrame(model_res).T
-
-            return df
-        return pd.DataFrame()

data/model_handler.py

@@ -1,98 +1,117 @@
 import json
 import os
+from typing import Any, Dict
 
 import pandas as pd
-from git import Repo
+from huggingface_hub import HfApi, hf_hub_download, metadata_load
 
-from .dataset_handler import VIDORE_V1_MTEB_NAMES, VIDORE_V2_MTEB_NAMES, get_datasets_nickname
+from .dataset_handler import VIDORE_DATASETS_KEYWORDS, get_datasets_nickname
+
+BLOCKLIST = ["impactframes"]
 
 
 class ModelHandler:
+    def __init__(self, model_infos_path="model_infos.json"):
+        self.api = HfApi()
+        self.model_infos_path = model_infos_path
+        self.model_infos = self._load_model_infos()
 
-    def __init__(self):
-        self.model_infos = {}
+    def _load_model_infos(self) -> Dict:
+        if os.path.exists(self.model_infos_path):
+            with open(self.model_infos_path) as f:
+                return json.load(f)
+        return {}
 
-    @staticmethod
-    def get_folders(dir_path):
-        return sorted([
-            path_
-            for path_ in os.listdir(dir_path)
-            if os.path.isdir(os.path.join(dir_path, path_))
-        ])
+    def _save_model_infos(self):
+        with open(self.model_infos_path, "w") as f:
+            json.dump(self.model_infos, f)
+
+    def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
+        return "metadata" in results and "metrics" in results
 
     def get_vidore_data(self, metric="ndcg_at_5"):
-        repo_url = "https://github.com/embeddings-benchmark/results.git"
-        local_path = "./results"
-        folder_of_interest = "results"
-
-        if os.path.exists(local_path):
-            repo = Repo(local_path)
-            origin = repo.remotes.origin
-            origin.pull()
-        else:
-            Repo.clone_from(repo_url, local_path, depth=1)
-
-        model_names = self.get_folders(os.path.join(local_path, folder_of_interest))
-        for model_name in model_names:
-            revisions = self.get_folders(os.path.join(local_path, folder_of_interest, model_name))
-            first_revision = revisions[0]
-            result_filenames = [
-                result_filename
-                for result_filename in os.listdir(os.path.join(local_path, folder_of_interest, model_name, first_revision))
-                # if result_filename.endswith(".json") and result_filename != "model_meta.json"
-            ]
-            if "model_meta.json" in result_filenames:
-                with open(os.path.join(local_path, folder_of_interest, model_name, first_revision, "model_meta.json"), "r") as f:
-                    meta = json.load(f)
+        models = self.api.list_models(filter="vidore")
+        repositories = [model.modelId for model in models]  # type: ignore
+
+        for repo_id in repositories:
+            org_name = repo_id.split("/")[0]
+            if org_name in BLOCKLIST:
+                continue
+
+            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith("_metrics.json") or f == "results.json"]
+
+            if len(files) == 0:
+                continue
             else:
-                meta = {}
-            results = {}
-            if all(f"{v1_dataset_name}.json" in result_filenames for v1_dataset_name in VIDORE_V1_MTEB_NAMES):
-                for v1_dataset_name in VIDORE_V1_MTEB_NAMES:
-                    with open(os.path.join(local_path, folder_of_interest, model_name, first_revision, f"{v1_dataset_name}.json"), "r") as f:
-                        results[v1_dataset_name] = json.load(f)
-            if all(f"{v2_dataset_name}.json" in result_filenames for v2_dataset_name in VIDORE_V2_MTEB_NAMES):
-                for v2_dataset_name in VIDORE_V2_MTEB_NAMES:
-                    with open(os.path.join(local_path, folder_of_interest, model_name, first_revision, f"{v2_dataset_name}.json"), "r") as f:
-                        results[v2_dataset_name] = json.load(f)
-                if model_name not in self.model_infos:
-                    self.model_infos[model_name] = {}
-                self.model_infos[model_name] = {"meta": meta, "results": results}
-
-    def filter_models_by_benchmark(self, benchmark_version=1):
-        filtered_model_infos = {}
-        keywords = VIDORE_V1_MTEB_NAMES if benchmark_version == 1 else VIDORE_V2_MTEB_NAMES
-
-        for model, info in self.model_infos.items():
-            results = info["results"]
-            if any(any(keyword in dataset for keyword in keywords) for dataset in results.keys()):
-                filtered_model_infos[model] = info
-
-        return filtered_model_infos
-
-    def render_df(self, metric="ndcg_at_5", benchmark_version=1):
+                for file in files:
+                    if file.endswith("results.json"):
+                        model_name = repo_id.replace("/", "_")
+                    else:
+                        model_name = file.split("_metrics.json")[0]
+
+                    if model_name not in self.model_infos:
+                        readme_path = hf_hub_download(repo_id, filename="README.md")
+                        meta = metadata_load(readme_path)
+                        try:
+                            result_path = hf_hub_download(repo_id, filename=file)
+
+                            with open(result_path) as f:
+                                results = json.load(f)
+
+                            if self._are_results_in_new_vidore_format(results):
+                                metadata = results["metadata"]
+                                results = results["metrics"]
+
+                            self.model_infos[model_name] = {"meta": meta, "results": results}
+                        except Exception as e:
+                            print(f"Error loading {model_name} - {e}")
+                            continue
+
+        # self._save_model_infos()
+
         model_res = {}
-        filtered_model_infos = self.filter_models_by_benchmark(benchmark_version)
-        if len(filtered_model_infos) > 0:
-            for model in filtered_model_infos.keys():
-                res = filtered_model_infos[model]["results"]
+        if len(self.model_infos) > 0:
+            for model in self.model_infos.keys():
+                res = self.model_infos[model]["results"]
                 dataset_res = {}
-                keywords = VIDORE_V1_MTEB_NAMES if benchmark_version == 1 else VIDORE_V2_MTEB_NAMES
-                if "n_parameters" in filtered_model_infos[model]["meta"]:
-                    try:
-                        dataset_res["Model Size (Million Parameters)"] = filtered_model_infos[model]["meta"]["n_parameters"] // 1_000_000
-                    except TypeError:
-                        dataset_res["Model Size (Million Parameters)"] = -1
-                else:
-                    dataset_res["Model Size (Million Parameters)"] = -1
                 for dataset in res.keys():
-                    if not any(keyword in dataset for keyword in keywords):
+                    # for each keyword check if it is in the dataset name if not continue
+                    if not any(keyword in dataset for keyword in VIDORE_DATASETS_KEYWORDS):
+                        print(f"{dataset} not found in ViDoRe datasets. Skipping ...")
                         continue
+
                     dataset_nickname = get_datasets_nickname(dataset)
-                    dataset_res[dataset_nickname] = res[dataset]["scores"]["test"][0][metric]
+                    dataset_res[dataset_nickname] = res[dataset][metric]
                 model_res[model] = dataset_res
 
             df = pd.DataFrame(model_res).T
 
             return df
         return pd.DataFrame()
+
+    @staticmethod
+    def add_rank(df):
+        df.fillna(0.0, inplace=True)
+        cols_to_rank = [
+            col
+            for col in df.columns
+            if col
+            not in [
+                "Model",
+                "Model Size (Million Parameters)",
+                "Memory Usage (GB, fp32)",
+                "Embedding Dimensions",
+                "Max Tokens",
+            ]
+        ]
+        if len(cols_to_rank) == 1:
+            df.sort_values(cols_to_rank[0], ascending=False, inplace=True)
+        else:
+            df.insert(len(df.columns) - len(cols_to_rank), "Average", df[cols_to_rank].mean(axis=1, skipna=False))
+            df.sort_values("Average", ascending=False, inplace=True)
+        df.insert(0, "Rank", list(range(1, len(df) + 1)))
+        # multiply values by 100 if they are floats and round to 1 decimal place
+        for col in df.columns:
+            if df[col].dtype == "float64":
+                df[col] = df[col].apply(lambda x: round(x * 100, 1))
+        return df

data/pipeline_handler.py

@@ -1,274 +0,0 @@
-import os
-import requests
-from typing import Dict, List, Optional
-
-import pandas as pd
-
-
-class PipelineHandler:
-    """Handler for ViDoRe v3 pipeline evaluation results from GitHub."""
-
-    def __init__(self):
-        self.pipeline_infos = {}
-        self.pipeline_aliases = {}  # Maps folder_name -> pipeline_alias for display
-        self.github_base_url = "https://raw.githubusercontent.com/illuin-tech/vidore-benchmark/main/results/metrics"
-        self.github_descriptions_base_url = (
-            "https://raw.githubusercontent.com/illuin-tech/vidore-benchmark/main/results/pipeline_descriptions"
-        )
-        self.available_datasets = []
-        self.available_languages = ["english"]  # Default languages available
-
-        # Setup GitHub authentication if token is available
-        self.github_token = os.environ.get("GITHUB_TOKEN")
-        self.headers = {}
-        if self.github_token:
-            self.headers["Authorization"] = f"token {self.github_token}"
-            print("GitHub token detected - using authenticated requests")
-
-    def get_pipeline_folders_from_github(self) -> List[str]:
-        """Get list of pipeline folders from GitHub API."""
-        api_url = "https://api.github.com/repos/illuin-tech/vidore-benchmark/contents/results/metrics"
-
-        try:
-            response = requests.get(api_url, headers=self.headers)
-            response.raise_for_status()
-            contents = response.json()
-
-            # Filter for directories only
-            folders = [item["name"] for item in contents if item["type"] == "dir"]
-            return sorted(folders)
-        except Exception as e:
-            print(f"Error fetching pipeline folders from GitHub: {e}")
-            return []
-
-    def get_dataset_files_from_github(self, pipeline_name: str) -> List[str]:
-        """Get list of dataset JSON files for a specific pipeline from GitHub API."""
-        api_url = f"https://api.github.com/repos/illuin-tech/vidore-benchmark/contents/results/metrics/{pipeline_name}"
-
-        try:
-            response = requests.get(api_url, headers=self.headers)
-            response.raise_for_status()
-            contents = response.json()
-
-            # Filter for JSON files that start with 'vidore_v3'
-            files = [
-                item["name"]
-                for item in contents
-                if item["type"] == "file" and item["name"].startswith("vidore_v3") and item["name"].endswith(".json")
-            ]
-            return sorted(files)
-        except Exception as e:
-            print(f"Error fetching dataset files from {pipeline_name}: {e}")
-            return []
-
-    def fetch_json_from_github(self, pipeline_name: str, filename: str) -> Optional[Dict]:
-        """Fetch a JSON file from GitHub raw content."""
-        url = f"{self.github_base_url}/{pipeline_name}/{filename}"
-
-        try:
-            response = requests.get(url, headers=self.headers)
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            print(f"Error fetching {filename} from {pipeline_name}: {e}")
-            return None
-
-    def fetch_pipeline_alias(self, pipeline_name: str) -> Optional[str]:
-        """Fetch the pipeline_alias from description.json for a pipeline.
-
-        Uses raw.githubusercontent.com to avoid API rate limits.
-        """
-        url = f"{self.github_descriptions_base_url}/{pipeline_name}/description.json"
-
-        try:
-            response = requests.get(url, headers=self.headers)
-            response.raise_for_status()
-            description = response.json()
-            return description.get("pipeline_alias")
-        except Exception as e:
-            print(f"Error fetching description for {pipeline_name}: {e}")
-            return None
-
-    def get_pipeline_data(self):
-        """Fetch all pipeline data from GitHub."""
-        pipeline_folders = self.get_pipeline_folders_from_github()
-        datasets_set = set()
-        languages_set = set(["overall"])
-
-        for pipeline_name in pipeline_folders:
-            # Get all dataset files for this pipeline
-            dataset_files = self.get_dataset_files_from_github(pipeline_name)
-
-            if not dataset_files:
-                continue
-
-            pipeline_data = {}
-            for filename in dataset_files:
-                results = self.fetch_json_from_github(pipeline_name, filename)
-                if results:
-                    # Extract dataset name from filename (remove vidore_v3_ prefix and .json suffix)
-                    dataset_name = filename.replace("vidore_v3_", "").replace(".json", "")
-                    datasets_set.add(dataset_name)
-                    pipeline_data[dataset_name] = results
-
-                    # Collect available languages
-                    if "aggregated_metrics" in results and "by_language" in results["aggregated_metrics"]:
-                        languages_set.update(results["aggregated_metrics"]["by_language"].keys())
-
-            if pipeline_data:
-                self.pipeline_infos[pipeline_name] = pipeline_data
-                # Fetch the pipeline alias for display (uses raw URL, not API)
-                alias = self.fetch_pipeline_alias(pipeline_name)
-                if alias:
-                    self.pipeline_aliases[pipeline_name] = alias
-
-        self.available_datasets = sorted(list(datasets_set))
-        self.available_languages = sorted(list(languages_set))
-
-    def calculate_cost_metric(self, pipeline_datasets: Dict) -> float:
-        """
-        Calculate a compute cost metric based on retrieval time across all datasets.
-        Returns cost in arbitrary units (could be refined based on actual compute costs).
-        """
-        total_time_s = 0
-
-        for dataset_name, dataset_data in pipeline_datasets.items():
-            if "aggregated_metrics" not in dataset_data:
-                continue
-
-            timing = dataset_data["aggregated_metrics"].get("timing", {})
-            total_time_ms = timing.get("total_retrieval_time_milliseconds", 0)
-            total_time_s += total_time_ms / 1000.0
-
-        # Simple cost model: assume $0.01 per second of compute (adjustable)
-        cost = total_time_s * 0.01
-
-        return round(cost, 4)
-
-    def extract_dataset_metrics(
-        self, pipeline_datasets: Dict, metric: str = "ndcg_cut_5", language: str = "english"
-    ) -> Dict[str, float]:
-        """
-        Extract metrics for individual datasets from the aggregated results.
-
-        Args:
-            pipeline_datasets: Dictionary mapping dataset names to their data
-            metric: The metric to extract (e.g., 'ndcg_at_5')
-            language: The language to filter by ('english' for English-only results, or specific language)
-
-        Returns:
-            Dictionary mapping dataset names to metric values
-        """
-        # Map metric names from UI format to API format
-        metric_mapping = {
-            "ndcg_at_1": "ndcg_cut_5",  # Using cut_5 as approximation
-            "ndcg_at_5": "ndcg_cut_5",
-            "ndcg_at_10": "ndcg_cut_10",
-            "ndcg_at_100": "ndcg_cut_100",
-            "recall_at_1": "recall_5",
-            "recall_at_5": "recall_5",
-            "recall_at_10": "recall_10",
-            "recall_at_100": "recall_100",
-        }
-
-        actual_metric = metric_mapping.get(metric, metric)
-        dataset_metrics = {}
-
-        for dataset_name, dataset_data in pipeline_datasets.items():
-            if "aggregated_metrics" not in dataset_data:
-                continue
-
-            aggregated = dataset_data["aggregated_metrics"]
-
-            # Get metrics for the specified language
-            if language == "overall":
-                metrics_data = aggregated.get("overall", {})
-            else:
-                metrics_data = aggregated.get("by_language", {}).get(language, {})
-
-            if metrics_data:
-                # Format dataset name for display
-                display_name = dataset_name.replace("_", " ").title()
-                dataset_metrics[display_name] = metrics_data.get(actual_metric, 0.0)
-
-        return dataset_metrics
-
-    def render_df(self, metric: str = "ndcg_at_5", language: str = "overall") -> pd.DataFrame:
-        """
-        Render a DataFrame with pipeline results.
-
-        Args:
-            metric: The metric to display (e.g., 'ndcg_at_5')
-            language: The language to filter by ('overall' for all languages, or specific language)
-
-        Returns:
-            DataFrame with columns: Pipeline Name, Compute Cost, Timing metrics, Dataset metrics
-        """
-        pipeline_res = {}
-
-        for pipeline_name, pipeline_datasets in self.pipeline_infos.items():
-            row_data = {}
-
-            # Aggregate time metrics across all datasets
-            total_time_ms = 0
-            total_queries = 0
-            indexing_time_ms = 0
-            search_time_ms = 0
-            num_datasets = 0
-
-            for dataset_name, dataset_data in pipeline_datasets.items():
-                if "aggregated_metrics" in dataset_data:
-                    timing = dataset_data["aggregated_metrics"].get("timing", {})
-                    total_time_ms += timing.get("total_retrieval_time_milliseconds", 0)
-                    total_queries += timing.get("num_queries", 0)
-                    indexing_time_ms += timing.get("indexing_throughput_ms_per_doc", 0)
-                    search_time_ms += timing.get("search_throughput_ms_per_query", 0)
-                    num_datasets += 1
-
-            if total_queries > 0:
-                if total_time_ms > 0:
-                    # row_data["Total retrieval time (s)"] = round(
-                    #     (total_time_ms / 1000), 0
-                    # )
-                    row_data["Indexing latency (s/doc)"] = (
-                        (indexing_time_ms / 1000) / num_datasets if indexing_time_ms > 0 else None
-                    )
-                    row_data["Search latency (s/query)"] = (
-                        (search_time_ms / 1000) / num_datasets if search_time_ms > 0 else None
-                    )
-                else:
-                    # row_data["Total retrieval time (s)"] = 0
-                    row_data["Indexing latency (s/doc)"] = 0
-                    row_data["Search latency (s/query)"] = 0
-            else:
-                # row_data["Total retrieval time (s)"] = -1
-                row_data["Indexing latency (s/doc)"] = -1
-                row_data["Search latency (s/query)"] = -1
-
-            # Add dataset metrics
-            dataset_metrics = self.extract_dataset_metrics(pipeline_datasets, metric, language)
-            row_data.update(dataset_metrics)
-
-            # Calculate average across datasets if there are multiple
-            if dataset_metrics:
-                row_data["Average Score"] = round(sum(dataset_metrics.values()) / len(dataset_metrics), 4)
-
-            # Use pipeline_alias for display if available, otherwise fallback to folder name
-            display_name = self.pipeline_aliases.get(pipeline_name, pipeline_name)
-            # Store folder name for link generation (will be used in utils.py)
-            row_data["_folder_name"] = pipeline_name
-            pipeline_res[display_name] = row_data
-
-        if pipeline_res:
-            df = pd.DataFrame(pipeline_res).T
-            # Reorder columns to have Average right after timing metrics
-            cols = list(df.columns)
-            if "Average Score" in cols:
-                cols.remove("Average Score")
-                # Insert Average after Search latency (s/query)
-                insert_pos = cols.index("Search latency (s/query)") + 1 if "Search latency (s/query)" in cols else 2
-                cols.insert(insert_pos, "Average Score")
-                df = df[cols]
-            return df
-
-        return pd.DataFrame()

requirements.txt

@@ -1,6 +0,0 @@
-gitpython
-gradio
-pydub
-huggingface_hub < 1.0.0
-plotly
-pandas