feat: add u-math vs mu-math

README.md

@@ -42,4 +42,4 @@ If you encounter problem on the space, don't hesitate to restart it to remove th
 You'll find 
 - the main table' columns names and properties in `src/display/utils.py`
 - the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -1,4 +1,5 @@
 import os
+from typing import Any
 
 import gradio as gr
 import pandas as pd
@@ -9,9 +10,11 @@ from src.about import CITATION_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TIT
 from src.populate import (
     MU_MATH_COLUMNS_DICT,
     U_MATH_COLUMNS_DICT,
+    U_MATH_AND_MU_MATH_COLUMNS_DICT,
     Field,
     get_mu_math_leaderboard_df,
     get_u_math_leaderboard_df,
+    get_joined_leaderboard_df,
 )
 
 
@@ -24,6 +27,7 @@ def restart_space():
 
 LEADERBOARD_U_MATH_DF = get_u_math_leaderboard_df()
 LEADERBOARD_MU_MATH_DF = get_mu_math_leaderboard_df()
+LEADERBOARD_U_MATH_MU_MATH_JOINED_DF = get_joined_leaderboard_df()
 
 
 def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) -> gr.components.Component:
@@ -79,6 +83,15 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
             filtered_df = full_df[full_df[columns_dict["model_size_symbol"].pretty_name] == query_symbol]
             return filtered_df[current_df.columns]
 
+    def filter_dataframe_by_model_family(
+        full_df: pd.DataFrame, current_df: pd.DataFrame, filter_name: str,
+    ) -> pd.DataFrame:
+        if filter_name == "All":
+            return full_df[current_df.columns]
+        else:
+            filtered_df = full_df[full_df[columns_dict["model_family"].pretty_name] == filter_name]
+            return filtered_df[current_df.columns]
+
     with gr.Column() as col:
         # Add the controls
         with gr.Accordion("➡️ See All Columns", open=False):
@@ -129,6 +142,14 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
                 interactive=True,
             )
 
+            model_family_filter_selector = gr.Radio(
+                label="Filter model families:",
+                choices=["All"] + list(dataframe[columns_dict["model_family"].pretty_name].unique()),
+                value="All",
+                elem_id="model-family-filter",
+                interactive=True,
+            )
+
         # create the hidden and visible dataframes to display
         hidden_leaderboard_df = gr.components.Dataframe(
             value=dataframe,
@@ -163,6 +184,11 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
             inputs=[hidden_leaderboard_df, leaderboard_df, model_size_filter_selector],
             outputs=[leaderboard_df],
         )
+        model_family_filter_selector.change(
+            fn=filter_dataframe_by_model_family,
+            inputs=[hidden_leaderboard_df, leaderboard_df, model_family_filter_selector],
+            outputs=[leaderboard_df],
+        )
         for tag, button in all_tags.items():
             button.click(
                 fn=filter_dataframe_by_selected_tag_columns,
@@ -206,7 +232,22 @@ with demo:
                 tooltip=[MU_MATH_COLUMNS_DICT["full_model_name"].pretty_name, MU_MATH_COLUMNS_DICT["mu_math_f1"].pretty_name],
             )
 
-        with gr.TabItem("📝 About", elem_id="about-tab-table", id=2):
+        with gr.TabItem("📊 U-MATH vs μ-MATH", elem_id="u-math-vs-mu-math-tab-table", id=2):
+            leaderboard_aggregated = init_leaderboard(LEADERBOARD_U_MATH_MU_MATH_JOINED_DF, U_MATH_AND_MU_MATH_COLUMNS_DICT)
+            gr.ScatterPlot(
+                value=LEADERBOARD_U_MATH_MU_MATH_JOINED_DF,
+                title="U-MATH Accuracy (Solving) vs μ-MATH F1 Score (Judging)",
+                x=U_MATH_AND_MU_MATH_COLUMNS_DICT["u_math_acc"].pretty_name,
+                y=U_MATH_AND_MU_MATH_COLUMNS_DICT["mu_math_f1"].pretty_name,
+                color=U_MATH_AND_MU_MATH_COLUMNS_DICT["model_family"].pretty_name,
+                tooltip=[
+                    U_MATH_AND_MU_MATH_COLUMNS_DICT["full_model_name"].pretty_name,
+                    U_MATH_AND_MU_MATH_COLUMNS_DICT["u_math_text_acc"].pretty_name,
+                    U_MATH_AND_MU_MATH_COLUMNS_DICT["u_math_visual_acc"].pretty_name,
+                ],
+            )
+
+        with gr.TabItem("📝 About", elem_id="about-tab-table", id=3):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
         citation_button = gr.Textbox(
@@ -222,4 +263,4 @@ scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=60 * 60)
 scheduler.start()
 # demo.queue(default_concurrency_limit=40).launch(ssr_mode=False)
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(allowed_paths=[".cache"])

src/populate.py

@@ -3,26 +3,36 @@ import os
 from dataclasses import dataclass, field
 
 import pandas as pd
-from huggingface_hub import model_info
+from huggingface_hub import model_info, ModelInfo
 from transformers import AutoConfig
 
 
 UNKNOWN_MODEL_SHOW_SIZE = 150
 
 
-def is_model_on_hub(
-    model_name: str, revision: str, token: str = None, trust_remote_code=False
-) -> tuple[bool, str | None, str | None]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+def get_hf_model_info_card_or_none(model_name: str) -> ModelInfo | None:
     try:
-        config = AutoConfig.from_pretrained(
-            model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
-        )
-        return True, None, config
+        info = model_info(repo_id=model_name)
+        return info
+    except Exception:
+        return None
+
+
+def get_hf_hub_config_or_none(model_name: str) -> AutoConfig | None:
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision="main", trust_remote_code=True)
+        return config
     except Exception:
-        return False, "was not found on hub!", None
+        return None
 
 
+def get_hf_model_info_card_or_none(model_name: str) -> ModelInfo | None:
+    try:
+        info = model_info(repo_id=model_name)
+        return info
+    except Exception:
+        return None
+
 def model_size_to_symbol(model_size_in_b_params: int | None) -> str:
     """Converts model size to a symbol"""
     if model_size_in_b_params is None or model_size_in_b_params == 0 or not model_size_in_b_params:
@@ -67,23 +77,31 @@ def get_hf_data_by_model_name(model_name: str) -> dict:
     elif 'deepseek' in model_name.lower():
         model_family = "DeepSeek"
 
-    still_on_hub, _, model_config = is_model_on_hub(model_name, "main", trust_remote_code=True)
-    if not still_on_hub and '/' in model_name:
+    print(model_name, model_family)
+    model_config = get_hf_hub_config_or_none(model_name)
+    model_info_card = get_hf_model_info_card_or_none(model_name)
+    print('model_config', type(model_config))
+    print('model_info_card', type(model_info_card))
+
+    # If model name is a path, try to get the model name from the hub
+    if '/' in model_name:
         print(f"Model {model_name} is not on the hub, try unsloth/...")
         model_name = "unsloth/" + model_name.split("/")[-1]
-        still_on_hub, _, model_config = is_model_on_hub(model_name, "main", trust_remote_code=True)
+        if not model_config:
+            model_config = get_hf_hub_config_or_none(model_name)
+        if not model_info_card:
+            model_info_card = get_hf_model_info_card_or_none(model_name)
 
     architecture = "Unknown"
-    if model_config is not None:
+    if model_config:
         architectures = getattr(model_config, "architectures", None)
         if architectures:
             architecture = ";".join(architectures)
 
     num_params = None
-    if still_on_hub:
-        info = model_info(repo_id=model_name)
+    if model_info_card:
         try:
-            num_params = round(info.safetensors["total"] / 1e9, 1)
+            num_params = round(model_info_card.safetensors["total"] / 1e9, 1)
         except Exception as e:
             print("SafeTensors not found in", model_name, e)
             if 'Pixtral-12B' in model_name:
@@ -94,15 +112,13 @@ def get_hf_data_by_model_name(model_name: str) -> dict:
         print("num_params", model_name, num_params)
 
     model_url = None
-    if still_on_hub:
+    if model_config or model_info_card:
         model_url = f"https://huggingface.co/{model_name}"
 
     model_license = "Unknown"
-    if model_config is not None:
-        info = model_info(repo_id=model_name)
-        # print(info.card_data)
-        model_license = info.card_data["license_name"]
-        model_license_link = info.card_data["license_link"]
+    if model_info_card:
+        model_license = model_info_card.card_data["license_name"]
+        model_license_link = model_info_card.card_data["license_link"]
         if model_license_link:
             model_license = f"[{model_license}]({model_license_link})"
         if not model_license:
@@ -110,7 +126,7 @@ def get_hf_data_by_model_name(model_name: str) -> dict:
 
     return {
         "model_architecture": architecture,
-        "model_type": "Open-Weights" if still_on_hub else "Proprietary",
+        "model_type": "Open-Weights" if model_info_card else "Proprietary",
         "model_size": num_params if num_params else None,
         "model_url": model_url,
         "model_license": model_license,
@@ -131,7 +147,7 @@ class Field:
 MODEL_COLUMNS_DICT = {
     "model_type_symbol": Field("T", "str", never_hidden=True),
     "model_size_symbol": Field("S", "str", never_hidden=True),
-    "full_model_name": Field("Full Model Name", "markdown", fully_hidden=True),
+    "full_model_name": Field("Full Model Name", "markdown", fully_hidden=True, displayed_by_default=False),
     "model_name": Field("Model Name", "markdown", never_hidden=True),
     "model_type": Field("Type", "str", displayed_by_default=False),
     "model_size": Field("#Params (B)", "number", displayed_by_default=False),
@@ -210,6 +226,21 @@ MU_MATH_COLUMNS_DICT = {
     "Qwen2.5-72B-Instruct_ppv": Field("Qwen2.5-72B Subset PPV", "number", displayed_by_default=False),
     "Qwen2.5-72B-Instruct_npv": Field("Qwen2.5-72B Subset NPV", "number", displayed_by_default=False),
 }
+U_MATH_AND_MU_MATH_COLUMNS_DICT = {
+    "u_math_rank": Field("U-MATH Rank", "number", never_hidden=True),
+    "mu_math_rank": Field("μ-MATH Rank", "number", never_hidden=True),
+    **MODEL_COLUMNS_DICT,
+    "u_math_acc": Field("U-MATH Acc", "number", tags=["main", "u_math", "mu_math"]),
+    "u_math_text_acc": Field("U-MATH Text Acc", "number", displayed_by_default=False, tags=["u_math"]),
+    "u_math_visual_acc": Field("U-MATH Visual Acc", "number", displayed_by_default=False, tags=["u_math"]),
+    "judge_model_name": Field("Judge Model Name", "markdown", displayed_by_default=False),
+    "extract_model_name": Field("Extract Model Name", "markdown", displayed_by_default=False),
+    "mu_math_f1": Field("μ-MATH F1", "number", tags=["main", "u_math", "mu_math"]),
+    "mu_math_tpr": Field("μ-MATH TPR", "number", displayed_by_default=False, tags=["mu_math"]),
+    "mu_math_tnr": Field("μ-MATH TNR", "number", displayed_by_default=False, tags=["mu_math"]),
+    "mu_math_ppv": Field("μ-MATH PPV", "number", displayed_by_default=False, tags=["mu_math"]),
+    "mu_math_npv": Field("μ-MATH NPV", "number", displayed_by_default=False, tags=["mu_math"]),
+}
 
 
 def load_json_data(json_path: str, main_col: str | None = None) -> pd.DataFrame:
@@ -226,10 +257,30 @@ def load_json_data(json_path: str, main_col: str | None = None) -> pd.DataFrame:
     return df
 
 
-def get_u_math_leaderboard_df() -> pd.DataFrame:
+def get_model_meta_info_df(model_full_names: list[str]) -> pd.DataFrame:
+    """Given a list of model names, returns a dataframe with model meta info"""
+    model_to_meta_dict = {
+        model_name: get_hf_data_by_model_name(model_name) for model_name in model_full_names
+    }
+
+    df = pd.DataFrame.from_dict(model_to_meta_dict, orient="index")
+    df["model_type_symbol"] = df["model_type"].apply(model_type_to_symbol)
+    df["model_size_symbol"] = df["model_size"].apply(model_size_to_symbol)
+    df["model_size_including_unknown"] = df["model_size"].apply(lambda x: x if x and pd.notna(x) else UNKNOWN_MODEL_SHOW_SIZE).astype(float)
+    df["full_model_name"] = df.index
+    df = df.reset_index(drop=True)
+    df["model_name"] = df["full_model_name"].apply(
+        lambda x: f"[{x}]({url})" if (url := model_to_meta_dict[x]["model_url"]) else x
+    )
+
+    return df
+
+
+def get_u_math_leaderboard_df(use_pretty_names: bool = True, add_meta: bool = True) -> pd.DataFrame:
     """Creates a dataframe from json with U-MATH eval results"""
     json_path = os.path.join("data", "u_math_eval_results.json")
     df = load_json_data(json_path)
+    df = df.rename(columns={"model_name": "full_model_name"})
 
     # flatten list [x, y, z] in columns as ["_acc", "_text_acc", "_visual_acc"] suffixes for columns
     for col in [
@@ -251,33 +302,23 @@ def get_u_math_leaderboard_df() -> pd.DataFrame:
     df["rank"] = range(1, len(df) + 1)
 
     # populate with model info
-    model_to_meta_dict = {
-        model_name: get_hf_data_by_model_name(model_name) for model_name in df["model_name"].unique()
-    }
-    df["model_architecture"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_architecture"])
-    df["model_license"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_license"])
-    df["model_type"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_type"])
-    df["model_type_symbol"] = df["model_type"].apply(model_type_to_symbol)
-    df["model_family"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_family"])
-    df["model_size"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_size"])
-    df["model_size_including_unknown"] = df["model_size"].apply(lambda x: x if x and pd.notna(x) else UNKNOWN_MODEL_SHOW_SIZE).astype(float)
-    df["model_size_symbol"] = df["model_size"].apply(model_size_to_symbol)
-    df["full_model_name"] = df["model_name"]
-    df["model_name"] = df["model_name"].apply(
-        lambda x: f"[{x}]({url})" if (url := model_to_meta_dict[x]["model_url"]) else x
-    )
+    if add_meta:
+        df_meta = get_model_meta_info_df(df["full_model_name"].unique())
+        df = pd.merge(df, df_meta, on=["full_model_name"], how="left")
 
     # convert to pretty names and sort columns by order in dict
-    df = df[U_MATH_COLUMNS_DICT.keys()]
-    df = df.rename(columns={key: col.pretty_name for key, col in U_MATH_COLUMNS_DICT.items() if key in df.columns})
+    if use_pretty_names:
+        df = df[U_MATH_COLUMNS_DICT.keys()]
+        df = df.rename(columns={key: col.pretty_name for key, col in U_MATH_COLUMNS_DICT.items() if key in df.columns})
 
     return df
 
 
-def get_mu_math_leaderboard_df() -> pd.DataFrame:
+def get_mu_math_leaderboard_df(use_pretty_names: bool = True, add_meta: bool = True) -> pd.DataFrame:
     """Creates a dataframe from json with mu-MATH eval results"""
     json_path = os.path.join("data", "mu_math_eval_results.json")
     df = load_json_data(json_path)
+    df = df.rename(columns={"model_name": "full_model_name"})
 
     # Calculate columns with prefixes f1, tpr, tnr, ppv, npv
     for col in ["mu_math", "GPT-4o", "Gemini-1.5-Pro", "Llama-3.1-70B-Instruct", "Qwen2.5-72B-Instruct"]:
@@ -306,24 +347,42 @@ def get_mu_math_leaderboard_df() -> pd.DataFrame:
     df["rank"] = range(1, len(df) + 1)
 
     # populate with model info
-    model_to_meta_dict = {
-        model_name: get_hf_data_by_model_name(model_name) for model_name in df["model_name"].unique()
-    }
-    df["model_architecture"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_architecture"])
-    df["model_license"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_license"])
-    df["model_type"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_type"])
-    df["model_type_symbol"] = df["model_type"].apply(model_type_to_symbol)
-    df["model_family"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_family"])
-    df["model_size"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_size"])
-    df["model_size_including_unknown"] = df["model_size"].apply(lambda x: x if x and pd.notna(x) else UNKNOWN_MODEL_SHOW_SIZE).astype(float)
-    df["model_size_symbol"] = df["model_size"].apply(model_size_to_symbol)
-    df["full_model_name"] = df["model_name"]
-    df["model_name"] = df["model_name"].apply(
-        lambda x: f"[{x}]({url})" if (url := model_to_meta_dict[x]["model_url"]) else x
-    )
+    if add_meta:
+        df_meta = get_model_meta_info_df(df["full_model_name"].unique())
+        df = pd.merge(df, df_meta, on=["full_model_name"], how="left")
+
+    # convert to pretty names and sort columns by order in dict
+    if use_pretty_names:
+        df = df[MU_MATH_COLUMNS_DICT.keys()]
+        df = df.rename(columns={key: col.pretty_name for key, col in MU_MATH_COLUMNS_DICT.items() if key in df.columns})
+
+    return df
+
+
+def get_joined_leaderboard_df(use_pretty_names: bool = True, add_meta: bool = True) -> pd.DataFrame:
+    """Creates a dataframe from json with U-MATH and mu-MATH eval results"""
+    u_math_df = get_u_math_leaderboard_df(use_pretty_names=False, add_meta=False)
+    u_math_df = u_math_df.rename(columns={"rank": "u_math_rank"})
+    mu_math_df = get_mu_math_leaderboard_df(use_pretty_names=False, add_meta=False)
+    mu_math_df = mu_math_df.rename(columns={"rank": "mu_math_rank"})
+    assert set(u_math_df.columns).intersection(set(mu_math_df.columns)) == {"full_model_name"}, f"Columns overlap in {u_math_df.columns} and {mu_math_df.columns}"
+
+    # merge U-MATH and mu-MATH dataframes
+    df = pd.merge(u_math_df, mu_math_df, on=["full_model_name"], how="inner", suffixes=("", ""))
+
+    # sort by rank on u_math
+    df = df.sort_values(by=["u_math_rank"], ascending=True)
+
+    # add meta info
+    if add_meta:
+        df_meta = get_model_meta_info_df(df["full_model_name"].unique())
+        df = pd.merge(df, df_meta, on=["full_model_name"], how="left")
 
     # convert to pretty names and sort columns by order in dict
-    df = df[MU_MATH_COLUMNS_DICT.keys()]
-    df = df.rename(columns={key: col.pretty_name for key, col in MU_MATH_COLUMNS_DICT.items() if key in df.columns})
+    if use_pretty_names:
+        df = df[U_MATH_AND_MU_MATH_COLUMNS_DICT.keys()]
+        df = df.rename(
+            columns={key: col.pretty_name for key, col in U_MATH_AND_MU_MATH_COLUMNS_DICT.items() if key in df.columns}
+        )
 
     return df