core bench outline

app.py

@@ -281,9 +281,9 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
 </style>
 
 <header class="hal-header">
-    <h1 class="hal-title">Holistic Agent Leaderboard (HAL)</h1>
+    <h1 class="hal-title">CORE-Bench Leaderboard</h1>
     <p class="hal-subtitle">
-        A standardized, cost-aware, and third-party leaderboard for evaluating agents.
+            A leaderboard evaluating agent performance on CORE-Bench.
     </p>
 </header>""")
     gr.HTML("""
@@ -300,11 +300,11 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
         background-color: #ffffff;
         border-radius: 10px;
         margin: 0 15px;
-        text-align: left;
+        text-align: center;
         box-shadow: 0 6px 12px rgba(0, 0, 0, 0.1);
         display: flex;
         flex-direction: column;
-        align-items: flex-start;
+        align-items: center;
         border-top: 5px solid #3498db;
         transition: transform 0.3s ease, box-shadow 0.3s ease;
     }
@@ -316,7 +316,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
         font-size: 1.2em;
         font-weight: bold;
         color: #1b9e77;
-        margin-bottom: 10px;
+        margin-bottom: 15px;
         text-transform: uppercase;
         letter-spacing: 1px;
     }
@@ -328,128 +328,52 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
         line-height: 1.6;
         color: #333;
     }
+    .button {
+        margin: 15px 0;
+        padding: 10px 20px;
+        font-size: 1em;
+        font-weight: bold;
+        color: #fff;
+        background-color: #3498db;
+        border: none;
+        border-radius: 5px;
+        text-decoration: none;
+        display: inline-flex;
+        align-items: center;
+        transition: background-color 0.3s ease;
+    }
+    .button:hover {
+        background-color: #2980b9;
+    }
+    .button img {
+        margin-right: 8px;
+        height: 20px;
+    }
 </style>
 
 <div class="feature-row">
     <div class="feature-column">
-        <div class="feature-keyword">Standardized</div>
+        <div class="feature-keyword">Paper</div>
         <div class="feature-content">
-            <p class="feature-description">Evaluations across agent benchmarks are all recorded to a single leaderboard that evaluates every listed agent in the same way.</p>
+            <a href="https://arxiv.org/abs/2409.11363" class="button">
+                <img src="https://example.com/favicon-paper.png" alt="Paper Icon"> View Paper
+            </a>
         </div>
     </div>
     <div class="feature-column">
-        <div class="feature-keyword">Cost-controlled</div>
+        <div class="feature-keyword">Github</div>
         <div class="feature-content">
-            <p class="feature-description">For downstream users, understanding the cost of running agents is a significant need for adoption. For agent developers, cost-controlled evaluations help develop accurate baselines.</p>
+            <a href="https://github.com/siegelz/core-bench" class="button">
+                <img src="https://example.com/favicon-github.png" alt="Github Icon"> View Github
+            </a>
         </div>
     </div>
     <div class="feature-column">
-        <div class="feature-keyword">Third-party</div>
+        <div class="feature-keyword">Dataset</div>
         <div class="feature-content">
-            <p class="feature-description">Agent developers clearly have competing objectives in reporting accuracy: they want to achieve state-of-the-art performance.</p>
-        </div>
-    </div>
-</div>
-<style>
-    .section-heading {
-        font-size: 1.8em;
-        font-weight: bold;
-        color: #2c3e50;
-        margin-top: 40px;
-        margin-bottom: 20px;
-        text-align: left;
-    }
-    .user-types-container {
-        display: grid;
-        grid-template-columns: repeat(2, 1fr);
-        gap: 20px;
-        margin-top: 20px;
-    }
-    .user-type {
-        background-color: #ffffff;
-        border-radius: 10px;
-        padding: 25px;
-        box-shadow: 0 6px 12px rgba(0, 0, 0, 0.1);
-        transition: transform 0.3s ease, box-shadow 0.3s ease;
-        border-left: 5px solid #3498db;
-    }
-    .user-type:hover {
-        transform: translateY(-5px);
-        box-shadow: 0 5px 10px rgba(0, 0, 0, 0.15);
-    }
-    .user-type-title {
-        font-size: 1.2em;
-        font-weight: bold;
-        color: #3498db;
-        margin-bottom: 10px;
-    }
-    .user-type-description {
-        font-size: 0.95em;
-        line-height: 1.6;
-        color: #333;
-    }
-    .user-type-links a {
-        display: inline-block;
-        padding: 5px 12px;
-        margin-bottom: 5px;
-        background-color: #f0f4f8;
-        color: #2c3e50 !important; /* Force the color change */
-        text-decoration: none !important; /* Force remove underline */
-        border-radius: 15px;
-        font-size: 0.85em;
-        transition: all 0.3s ease;
-        border: 1px solid #e1e8ed;
-    }
-    .user-type-links a:hover {
-        background-color: #3498db;
-        color: white !important; /* Force the color change on hover */
-        transform: translateY(-2px);
-        box-shadow: 0 2px 5px rgba(52, 152, 219, 0.2);
-        text-decoration: none !important; /* Ensure no underline on hover */
-    }
-    .user-type-links a:visited {
-        color: #2c3e50 !important; /* Ensure visited links have the same color */
-    }
-    .user-type-links a::before {
-        content: "→";
-        margin-right: 5px;
-        font-size: 1.1em;
-    }
-</style>
-
-<h2 class="section-heading">Who is it for?</h2>
-<p>We see HAL being useful for four types of users:</p>
-
-<div class="user-types-container">
-    <div class="user-type">
-        <h3 class="user-type-title">Downstream Users & Procurers</h3>
-        <p class="user-type-description">Customers looking to deploy agents can get visibility into existing benchmarks, know developers building useful agents, and identify the state of the art for both cost and accuracy for their tasks of interest.</p>
-        <div class="user-type-links">
-            <a href="#leaderboards">Leaderboards</a>
-        </div>
-    </div>
-    <div class="user-type">
-        <h3 class="user-type-title">Agent Benchmark Developers</h3>
-        <p class="user-type-description">Reporting results on a centralized leaderboard could allow improved visibility into agent benchmarks that measure real-world utility.</p>
-        <div class="user-type-links">
-            <a href="#benchmark-submission">Add a Benchmark</a>
-        </div>
-    </div>
-    <div class="user-type">
-        <h3 class="user-type-title">Agent Developers</h3>
-        <p class="user-type-description">HAL allows for easy reproduction of past agents, clear comparison with past baselines, and a straightforward way to compete on a leaderboard.</p>
-        <div class="user-type-links">
-            <a href="#agent-submission">Submit an Agent</a>
-            <a href="#leaderboards">Leaderboards</a>
-            <a href="#reproduction-guide">Reproduction Guide</a>
-        </div>
-    </div>
-    <div class="user-type">
-        <h3 class="user-type-title">Safety Researchers</h3>
-        <p class="user-type-description">Understanding agent capabilities on real-world safety threats and their associated costs is crucial. For example, Cybench evaluations could provide insights into agent performance and affordability for potential adversaries.</p>
-        <div class="user-type-links">
-            <a href="#cybench-results">Cybench Leaderboard (coming soon)</a>
-            <a href="#agent-monitor">Agent Monitor</a>
+            <a href="https://huggingface.co/datasets/siegelz/core-bench" class="button">
+                <img src="https://example.com/favicon-dataset.png" alt="Dataset Icon"> View Dataset
+            </a>
         </div>
     </div>
 </div>
@@ -459,8 +383,16 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
 """)
     
     with gr.Tabs() as tabs:
-        with gr.Tab("USACO"):
-            gr.Markdown("""The USA Computing Olympiad (USACO) is a computer programming competition for pre-college students. This benchmark evaluates the performance of AI agents on a set of 307 USACO tasks. The agents are evaluated based on the number of tasks correctly solved.""")
+        with gr.Tab("CORE-Bench"):
+            gr.Markdown("""
+                        CORE-Bench evaluates the ability of agents to computationally reproduce the results of published scientific papers. Agents are given the codebase of a paper and must install all libraries and dependencies, run the code, and read through the output and figures to answer questions about the paper. The benchmark has tasks at three difficulty levels:
+                        
+                        <b>CORE-Bench-Easy</b>: The agent is given the output of the code and must answer questions about the output without running any code. To answer questions, agents must navigate through the terminal output as well as files and figures generated by the code.
+
+                        <b>CORE-Bench-Medium</b>: The agent is given a Dockerfile and instructions on how to use the Dockerfile to fully reproduce the paper. This level mainly evaluates agents ability to use and interact with the terminal. The agent must then answer questions about the output of the code, as in the above level.
+
+                        <b>CORE-Bench-Hard</b>: The agent is given the codebase of the paper and must install all libraries and dependencies, run the code, and read through the output and figures to answer questions about the paper. This level is most akin to fully reproducing a paper and is the most realistic and challenging level.
+                        """)
             with gr.Row():
                 with gr.Column(scale=2):
                     Leaderboard(
@@ -492,643 +424,6 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
             outputs=[task_success_heatmap]
             )
 
-            gr.HTML("""
-            <style>
-                .grouped-section {
-                    border: 2px solid #dee2e6; /* Color matching unactivated tabs */
-                    border-radius: 10px;
-                    padding: 30px;
-                    margin-top: 40px;
-                    margin-bottom: 40px;
-                    position: relative;
-                }
-
-                .grouped-section-title {
-                    font-size: 1.7em;
-                    font-weight: bold;
-                    color: #2c3e50;
-                    margin-bottom: 20px;
-                    padding-bottom: 10px;
-                    border-bottom: 2px solid #dee2e6;
-                }
-            </style>
-            """)
-            with gr.Group(elem_classes=["grouped-section"]):
-                gr.Markdown("# Agent monitor", elem_classes=["grouped-section-title"], elem_id="agent-monitor")
-                gr.Markdown('The agent monitor provides an overview of the recurring errors an agent makes as well as a summary of the steps the agent takes to solve a task. It currently consists of two main components:')
-
-                gr.HTML('<div style="height: 10px;"></div>')
-                gr.Markdown("## Failure report for each agent")
-                gr.Markdown('Select an agent to see why the agent fails to solve tasks correctly. Note that these descriptions (and the failure categories) are generated by LLM-based evaluations of the agent logs and may contain inaccuracies.')
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        failure_categories_overview = gr.Markdown()
-                
-                    with gr.Column(scale=1):
-                        failure_categories_chart = gr.Plot()
-
-                # Initialize the failure report agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                        outputs=[failure_report_agent_dropdown])
-                
-                # Update failure report when agent is selected
-                failure_report_agent_dropdown.change(update_failure_report,
-                                                    inputs=[failure_report_agent_dropdown, gr.Textbox(value="usaco", visible=False)],
-                                                    outputs=[failure_categories_overview, failure_categories_chart])
-
-                gr.HTML('<div style="height: 30px;"></div>')
-                gr.Markdown("## Task overview")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        task_dropdown = gr.Dropdown(label="Select USACO Task")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    task_overview = gr.Markdown()
-                with gr.Row():
-                    flow_chart = gr.Plot(label="Task Flow")
-
-                # Initialize the agent dropdown with the best agent
-                demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
-                demo.load(update_task_analysis, inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-
-                agent_dropdown.change(update_task_analysis, 
-                                    inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown],
-                                    outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-                task_dropdown.change(update_task_details,
-                                    inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown, task_dropdown],
-                                    outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
-            
-            gr.Markdown("## Raw predictions")
-            gr.Markdown('Select an agent to see the raw predictions made by the agent for each task. We also provide information on token usage for each call.')
-            with gr.Accordion("Expand to inspect raw predictions of agents...", open=False):
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        raw_agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        raw_task_dropdown = gr.Dropdown(label="Select Task")
-                    with gr.Column(scale=1):
-                        raw_step_dropdown = gr.Dropdown(label="Select Step")
-                with gr.Row():
-                    raw_call_details = gr.HTML()
-                
-                def update_raw_task_dropdown(agent_name):
-                    analyzed_traces = get_analyzed_traces(agent_name, "usaco")
-                    if not analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
-                    task_ids = list(analyzed_traces.keys())
-                    steps = analyzed_traces[task_ids[0]]['steps']
-                    return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(get_analyzed_traces(agent_name, "usaco")[task_ids[0]]['steps'][0], 0)
-
-                def update_raw_step_dropdown(agent_name, task_id):
-                    analyzed_traces = get_analyzed_traces(agent_name, "usaco")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
-                    steps = analyzed_traces[task_id]['steps']
-                    return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
-
-                def update_raw_call_details(agent_name, task_id, step_index):
-                    analyzed_traces = get_analyzed_traces(agent_name, "usaco")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return "No data available for this selection."
-                    steps = analyzed_traces[task_id]['steps']
-                    if step_index is None:
-                        return "Invalid step selection."
-                    step = steps[step_index]
-                    return format_call_info(step, step_index)
-
-                # Initialize the raw agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                        outputs=[raw_agent_dropdown])
-                demo.load(update_raw_task_dropdown,
-                        inputs=[raw_agent_dropdown],
-                        outputs=[raw_task_dropdown, raw_step_dropdown])
-                demo.load(update_raw_call_details,
-                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
-                        outputs=[raw_call_details])
-
-                raw_agent_dropdown.change(update_raw_task_dropdown, 
-                                        inputs=[raw_agent_dropdown], 
-                                        outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
-                raw_task_dropdown.change(update_raw_step_dropdown, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown], 
-                                        outputs=[raw_step_dropdown, raw_call_details])
-                raw_step_dropdown.change(update_raw_call_details, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
-                                        outputs=[raw_call_details])
-        
-        with gr.Tab("SWE-bench Verified"):
-            gr.Markdown("""SWE-bench is a dataset that tests systems' ability to solve GitHub issues automatically. Verified is a human-validated subset of 500 problems reviewed by software engineers. The  We are currently actively developing this platform and this benchmark is not fully implemented yet.""")
-            with gr.Row():
-                with gr.Column(scale=2):
-                    Leaderboard(
-                        value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'), ci_metrics=["Accuracy", "Total Cost"]),
-                        select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified"],
-                            cant_deselect=["Agent Name"],
-                            label="Select Columns to Display:",
-                        ),
-                        hide_columns=config.SWEBENCH_HIDE_COLUMNS,
-                        search_columns=config.SWEBENCH_SEARCH_COLUMNS,
-                    )
-                    gr.Markdown("""*Error ranges span from the lowest to highest observed values in repeated runs.*""", elem_classes=["text-right"])
-            with gr.Row():
-                gr.Markdown("### Accuracy vs. Cost for SWE-bench agents")
-            with gr.Row():
-                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified', aggregate=False), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
-            
-            gr.HTML('<div style="height: 30px;"></div>')
-            gr.Markdown("## Task success heatmap")
-            gr.Markdown("The task success heatmap shows which agent can solve which tasks. Agents are sorted by total accuracy (higher is better); tasks in SWE-bench are sorted by decreasing order of difficulty (tasks on the left are solved by the most agents; tasks on the right are solved by the least. For agents that have been run more than once, the run with the highest score is shown.")
-            with gr.Row():
-                task_success_heatmap = gr.Plot()
-            demo.load(
-            lambda: create_task_success_heatmap(
-                preprocessor.get_task_success_data('swebench_verified'),
-                'SWE-bench Verified'
-            ),
-            outputs=[task_success_heatmap]
-            )
-
-            gr.HTML("""
-            <style>
-                .grouped-section {
-                    border: 2px solid #dee2e6; /* Color matching unactivated tabs */
-                    border-radius: 10px;
-                    padding: 30px;
-                    margin-top: 40px;
-                    margin-bottom: 40px;
-                    position: relative;
-                }
-
-                .grouped-section-title {
-                    font-size: 1.7em;
-                    font-weight: bold;
-                    color: #2c3e50;
-                    margin-bottom: 20px;
-                    padding-bottom: 10px;
-                    border-bottom: 2px solid #dee2e6;
-                }
-            </style>
-            """)
-            with gr.Group(elem_classes=["grouped-section"]):
-                gr.Markdown("# Agent monitor", elem_classes=["grouped-section-title"], elem_id="agent-monitor")
-
-                gr.HTML('<div style="height: 10px;"></div>')
-                gr.Markdown("## Failure report for each agent")
-                gr.Markdown('Select an agent to see why the agent fails to solve tasks correctly. Note that these descriptions (and the failure categories) are generated by LLM-based evaluations of the agent logs and may contain inaccuracies.')
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        failure_categories_overview = gr.Markdown()
-                
-                    with gr.Column(scale=1):
-                        failure_categories_chart = gr.Plot()
-
-                # Initialize the failure report agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                        outputs=[failure_report_agent_dropdown])
-                
-                # Update failure report when agent is selected
-                failure_report_agent_dropdown.change(update_failure_report,
-                                                    inputs=[failure_report_agent_dropdown, gr.Textbox(value="swebench_verified", visible=False)],
-                                                    outputs=[failure_categories_overview, failure_categories_chart])
-
-                gr.HTML('<div style="height: 30px;"></div>')
-                gr.Markdown("## Task overview")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        task_dropdown = gr.Dropdown(label="Select SWE-bench Verified Task")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    task_overview = gr.Markdown()
-                with gr.Row():
-                    flow_chart = gr.Plot(label="Task Flow")
-
-                # Initialize the agent dropdown with the best agent
-                demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
-                demo.load(update_task_analysis, inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-
-                agent_dropdown.change(update_task_analysis, 
-                                    inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown],
-                                    outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-                task_dropdown.change(update_task_details,
-                                    inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown, task_dropdown],
-                                    outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
-            
-            gr.Markdown("## Raw predictions")
-            gr.Markdown('Select an agent to see the raw predictions made by the agent for each task. We also provide information on token usage for each call.')
-            with gr.Accordion("Expand to inspect raw predictions of agents...", open=False):
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        raw_agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        raw_task_dropdown = gr.Dropdown(label="Select Task")
-                    with gr.Column(scale=1):
-                        raw_step_dropdown = gr.Dropdown(label="Select Step")
-                with gr.Row():
-                    raw_call_details = gr.HTML()
-                
-                def update_raw_task_dropdown(agent_name):
-                    analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
-                    if not analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
-                    task_ids = list(analyzed_traces.keys())
-                    steps = analyzed_traces[task_ids[0]]['steps']
-                    return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(get_analyzed_traces(agent_name, "swebench_verified")[task_ids[0]]['steps'][0], 0)
-
-                def update_raw_step_dropdown(agent_name, task_id):
-                    analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
-                    steps = analyzed_traces[task_id]['steps']
-                    return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
-
-                def update_raw_call_details(agent_name, task_id, step_index):
-                    analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return "No data available for this selection."
-                    steps = analyzed_traces[task_id]['steps']
-                    if step_index is None:
-                        return "Invalid step selection."
-                    step = steps[step_index]
-                    return format_call_info(step, step_index)
-
-                # Initialize the raw agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                        outputs=[raw_agent_dropdown])
-                demo.load(update_raw_task_dropdown,
-                        inputs=[raw_agent_dropdown],
-                        outputs=[raw_task_dropdown, raw_step_dropdown])
-                demo.load(update_raw_call_details,
-                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
-                        outputs=[raw_call_details])
-
-                raw_agent_dropdown.change(update_raw_task_dropdown, 
-                                        inputs=[raw_agent_dropdown], 
-                                        outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
-                raw_task_dropdown.change(update_raw_step_dropdown, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown], 
-                                        outputs=[raw_step_dropdown, raw_call_details])
-                raw_step_dropdown.change(update_raw_call_details, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
-                                        outputs=[raw_call_details])
-        
-        
-        with gr.Tab("SWE-bench Lite"):
-            gr.Markdown("""SWE-bench is a dataset that tests systems' ability to solve GitHub issues automatically. Lite is a subset of 300 tasks of the original SWE-bench. We are currently actively developing this platform and this benchmark is not fully implemented yet.""")
-            with gr.Row():
-                with gr.Column(scale=2):
-                    Leaderboard(
-                        value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_lite'), ci_metrics=["Accuracy", "Total Cost"]),
-                        select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified"],
-                            cant_deselect=["Agent Name"],
-                            label="Select Columns to Display:",
-                        ),
-                        hide_columns=config.SWEBENCH_HIDE_COLUMNS,
-                        search_columns=config.SWEBENCH_SEARCH_COLUMNS,
-                    )
-                    gr.Markdown("""*Error ranges span from the lowest to highest observed values in repeated runs.*""", elem_classes=["text-right"])
-            with gr.Row():
-                gr.Markdown("### Accuracy vs. Cost for SWE-bench agents")
-            with gr.Row():
-                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_lite', aggregate=False), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
-            
-            gr.HTML('<div style="height: 30px;"></div>')
-            gr.Markdown("## Task success heatmap")
-            gr.Markdown("The task success heatmap shows which agent can solve which tasks. Agents are sorted by total accuracy (higher is better); tasks in SWE-bench are sorted by decreasing order of difficulty (tasks on the left are solved by the most agents; tasks on the right are solved by the least. For agents that have been run more than once, the run with the highest score is shown.")
-            with gr.Row():
-                task_success_heatmap = gr.Plot()
-            demo.load(
-            lambda: create_task_success_heatmap(
-                preprocessor.get_task_success_data('swebench_lite'),
-                'SWE-bench Lite'
-            ),
-            outputs=[task_success_heatmap]
-            )
-
-            gr.HTML("""
-            <style>
-                .grouped-section {
-                    border: 2px solid #dee2e6; /* Color matching unactivated tabs */
-                    border-radius: 10px;
-                    padding: 30px;
-                    margin-top: 40px;
-                    margin-bottom: 40px;
-                    position: relative;
-                }
-
-                .grouped-section-title {
-                    font-size: 1.7em;
-                    font-weight: bold;
-                    color: #2c3e50;
-                    margin-bottom: 20px;
-                    padding-bottom: 10px;
-                    border-bottom: 2px solid #dee2e6;
-                }
-            </style>
-            """)
-            with gr.Group(elem_classes=["grouped-section"]):
-                gr.Markdown("# Agent monitor", elem_classes=["grouped-section-title"], elem_id="agent-monitor")
-
-                gr.HTML('<div style="height: 10px;"></div>')
-                gr.Markdown("## Failure report for each agent")
-                gr.Markdown('Select an agent to see why the agent fails to solve tasks correctly. Note that these descriptions (and the failure categories) are generated by LLM-based evaluations of the agent logs and may contain inaccuracies.')
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        failure_categories_overview = gr.Markdown()
-                
-                    with gr.Column(scale=1):
-                        failure_categories_chart = gr.Plot()
-
-                # Initialize the failure report agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="swebench_lite", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                        outputs=[failure_report_agent_dropdown])
-                
-                # Update failure report when agent is selected
-                failure_report_agent_dropdown.change(update_failure_report,
-                                                    inputs=[failure_report_agent_dropdown, gr.Textbox(value="swebench_lite", visible=False)],
-                                                    outputs=[failure_categories_overview, failure_categories_chart])
-
-                gr.HTML('<div style="height: 30px;"></div>')
-                gr.Markdown("## Task overview")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        task_dropdown = gr.Dropdown(label="Select SWE-bench Lite Task")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    task_overview = gr.Markdown()
-                with gr.Row():
-                    flow_chart = gr.Plot(label="Task Flow")
-
-                # Initialize the agent dropdown with the best agent
-                demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="swebench_lite", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
-                demo.load(update_task_analysis, inputs=[gr.Textbox(value="swebench_lite", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-
-                agent_dropdown.change(update_task_analysis, 
-                                    inputs=[gr.Textbox(value="swebench_lite", visible=False), agent_dropdown],
-                                    outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-                task_dropdown.change(update_task_details,
-                                    inputs=[gr.Textbox(value="swebench_lite", visible=False), agent_dropdown, task_dropdown],
-                                    outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
-            
-            gr.Markdown("## Raw predictions")
-            gr.Markdown('Select an agent to see the raw predictions made by the agent for each task. We also provide information on token usage for each call.')
-            with gr.Accordion("Expand to inspect raw predictions of agents...", open=False):
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        raw_agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        raw_task_dropdown = gr.Dropdown(label="Select Task")
-                    with gr.Column(scale=1):
-                        raw_step_dropdown = gr.Dropdown(label="Select Step")
-                with gr.Row():
-                    raw_call_details = gr.HTML()
-                
-                def update_raw_task_dropdown(agent_name):
-                    analyzed_traces = get_analyzed_traces(agent_name, "swebench_lite")
-                    if not analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
-                    task_ids = list(analyzed_traces.keys())
-                    steps = analyzed_traces[task_ids[0]]['steps']
-                    return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(get_analyzed_traces(agent_name, "swebench_lite")[task_ids[0]]['steps'][0], 0)
-
-                def update_raw_step_dropdown(agent_name, task_id):
-                    analyzed_traces = get_analyzed_traces(agent_name, "swebench_lite")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
-                    steps = analyzed_traces[task_id]['steps']
-                    return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
-
-                def update_raw_call_details(agent_name, task_id, step_index):
-                    analyzed_traces = get_analyzed_traces(agent_name, "swebench_lite")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return "No data available for this selection."
-                    steps = analyzed_traces[task_id]['steps']
-                    if step_index is None:
-                        return "Invalid step selection."
-                    step = steps[step_index]
-                    return format_call_info(step, step_index)
-
-                # Initialize the raw agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="swebench_lite", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                        outputs=[raw_agent_dropdown])
-                demo.load(update_raw_task_dropdown,
-                        inputs=[raw_agent_dropdown],
-                        outputs=[raw_task_dropdown, raw_step_dropdown])
-                demo.load(update_raw_call_details,
-                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
-                        outputs=[raw_call_details])
-
-                raw_agent_dropdown.change(update_raw_task_dropdown, 
-                                        inputs=[raw_agent_dropdown], 
-                                        outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
-                raw_task_dropdown.change(update_raw_step_dropdown, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown], 
-                                        outputs=[raw_step_dropdown, raw_call_details])
-                raw_step_dropdown.change(update_raw_call_details, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
-                                        outputs=[raw_call_details])
-        
-        
-        
-        with gr.Tab("MLAgentBench"):
-            gr.Markdown("""MLAgentBench is a suite of end-to-end Machine Learning (ML) experimentation tasks, where the agent aims to take a given dataset and a machine learning task description and autonomously develop or improve an ML model. We are currently actively developing this platform and this benchmark is not fully implemented yet. In particular, we only include one agent and a subset of tasks for this benchmark.""")
-            with gr.Row():
-                with gr.Column(scale=2):
-                    Leaderboard(
-                        value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'mlagentbench')),
-                        select_columns=SelectColumns(
-                            default_selection=config.MLAGENTBENCH_ON_LOAD_COLUMNS + ["Verified"],
-                            cant_deselect=["Agent Name"],
-                            label="Select Columns to Display:",
-                        ),
-                        hide_columns=config.MLAGENTBENCH_HIDE_COLUMNS,
-                        search_columns=config.MLAGENTBENCH_SEARCH_COLUMNS,
-                    )
-                    gr.Markdown("""*Error ranges span from the lowest to highest observed values in repeated runs.*""", elem_classes=["text-right"])
-            with gr.Row():
-                gr.Markdown("### Accuracy vs. Cost for MLAgentBench agents")
-            with gr.Row():
-                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'mlagentbench', aggregate=False), "Total Cost", "Overall Score", "Total Cost (in USD)", "Overall Score", ["Agent Name"]))
-            
-            # gr.HTML('<div style="height: 30px;"></div>')
-            # gr.Markdown("## Task success heatmap")
-            # gr.Markdown("The task success heatmap shows which agent can solve which tasks. Agents are sorted by total accuracy (higher is better); tasks in USACO are sorted by decreasing order of difficulty (tasks on the left are solved by the most agents; tasks on the right are solved by the least. For agents that have been run more than once, the run with the highest score is shown.")
-            # with gr.Row():
-            #     task_success_heatmap = gr.Plot()
-            # demo.load(
-            # lambda: create_task_success_heatmap(
-            #     preprocessor.get_task_success_data('usaco'),
-            #     'USACO'
-            # ),
-            # outputs=[task_success_heatmap]
-            # )
-
-            gr.HTML("""
-            <style>
-                .grouped-section {
-                    border: 2px solid #dee2e6; /* Color matching unactivated tabs */
-                    border-radius: 10px;
-                    padding: 30px;
-                    margin-top: 40px;
-                    margin-bottom: 40px;
-                    position: relative;
-                }
-
-                .grouped-section-title {
-                    font-size: 1.7em;
-                    font-weight: bold;
-                    color: #2c3e50;
-                    margin-bottom: 20px;
-                    padding-bottom: 10px;
-                    border-bottom: 2px solid #dee2e6;
-                }
-            </style>
-            """)
-            with gr.Group(elem_classes=["grouped-section"]):
-                gr.Markdown("# Agent monitor", elem_classes=["grouped-section-title"], elem_id="agent-monitor")
-
-                # gr.HTML('<div style="height: 10px;"></div>')
-                # gr.Markdown("## Failure report for each agent")
-                # gr.Markdown('Select an agent to see why the agent fails to solve tasks correctly. Note that these descriptions (and the failure categories) are generated by LLM-based evaluations of the agent logs and may contain inaccuracies.')
-                # gr.HTML('<div style="height: 10px;"></div>')
-                # with gr.Row():
-                #     with gr.Column(scale=1):
-                #         failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
-                # gr.HTML('<div style="height: 10px;"></div>')
-                # with gr.Row():
-                #     with gr.Column(scale=1):
-                #         failure_categories_overview = gr.Markdown()
-                
-                #     with gr.Column(scale=1):
-                #         failure_categories_chart = gr.Plot()
-
-                # # Initialize the failure report agent dropdown with all agents
-                # demo.load(update_agent_dropdown, 
-                #         inputs=[gr.Textbox(value="mlagentbench", visible=False), gr.Textbox(value="Overall Score", visible=False)], 
-                #         outputs=[failure_report_agent_dropdown])
-                
-                # # Update failure report when agent is selected
-                # failure_report_agent_dropdown.change(update_failure_report,
-                #                                     inputs=[failure_report_agent_dropdown, gr.Textbox(value="mlagentbench", visible=False)],
-                #                                     outputs=[failure_categories_overview, failure_categories_chart])
-
-                gr.HTML('<div style="height: 30px;"></div>')
-                gr.Markdown("## Task overview")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        task_dropdown = gr.Dropdown(label="Select MLAgentBench Task")
-                gr.HTML('<div style="height: 10px;"></div>')
-                with gr.Row():
-                    task_overview = gr.Markdown()
-                with gr.Row():
-                    flow_chart = gr.Plot(label="Task Flow")
-
-                # Initialize the agent dropdown with the best agent
-                demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="mlagentbench", visible=False), gr.Textbox(value="Overall Score", visible=False)], outputs=[agent_dropdown])
-                demo.load(update_task_analysis, inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-
-                agent_dropdown.change(update_task_analysis, 
-                                    inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown],
-                                    outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-                task_dropdown.change(update_task_details,
-                                    inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown, task_dropdown],
-                                    outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
-            
-            gr.Markdown("## Raw predictions")
-            gr.Markdown('Select an agent to see the raw predictions made by the agent for each task. We also provide information on token usage for each call.')
-            with gr.Accordion("Expand to inspect raw predictions of agents...", open=False):
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        raw_agent_dropdown = gr.Dropdown(label="Select Agent")
-                    with gr.Column(scale=1):
-                        raw_task_dropdown = gr.Dropdown(label="Select Task")
-                    with gr.Column(scale=1):
-                        raw_step_dropdown = gr.Dropdown(label="Select Step")
-                with gr.Row():
-                    raw_call_details = gr.HTML()
-                
-                def update_raw_task_dropdown(agent_name):
-                    analyzed_traces = get_analyzed_traces(agent_name, "mlagentbench")
-                    if not analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
-                    task_ids = list(analyzed_traces.keys())
-                    steps = analyzed_traces[task_ids[0]]['steps']
-                    return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(get_analyzed_traces(agent_name, "mlagentbench")[task_ids[0]]['steps'][0], 0)
-
-                def update_raw_step_dropdown(agent_name, task_id):
-                    analyzed_traces = get_analyzed_traces(agent_name, "mlagentbench")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
-                    steps = analyzed_traces[task_id]['steps']
-                    return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
-
-                def update_raw_call_details(agent_name, task_id, step_index):
-                    analyzed_traces = get_analyzed_traces(agent_name, "mlagentbench")
-                    if not analyzed_traces or task_id not in analyzed_traces:
-                        return "No data available for this selection."
-                    steps = analyzed_traces[task_id]['steps']
-                    if step_index is None:
-                        return "Invalid step selection."
-                    step = steps[step_index]
-                    return format_call_info(step, step_index)
-
-                # Initialize the raw agent dropdown with all agents
-                demo.load(update_agent_dropdown, 
-                        inputs=[gr.Textbox(value="mlagentbench", visible=False), gr.Textbox(value="Overall Score", visible=False)], 
-                        outputs=[raw_agent_dropdown])
-                demo.load(update_raw_task_dropdown,
-                        inputs=[raw_agent_dropdown],
-                        outputs=[raw_task_dropdown, raw_step_dropdown])
-                demo.load(update_raw_call_details,
-                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
-                        outputs=[raw_call_details])
-
-                raw_agent_dropdown.change(update_raw_task_dropdown, 
-                                        inputs=[raw_agent_dropdown], 
-                                        outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
-                raw_task_dropdown.change(update_raw_step_dropdown, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown], 
-                                        outputs=[raw_step_dropdown, raw_call_details])
-                raw_step_dropdown.change(update_raw_call_details, 
-                                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
-                                        outputs=[raw_call_details])   
-        
-        with gr.Tab("About"):
-            gr.Markdown((Path(__file__).parent / "about.md").read_text())
 
     # Will trigger autoscaling of plots when tabs are switched
     tabs.select(fn=None, inputs=None, outputs=None, js="""
@@ -1145,10 +440,6 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
     gr.HTML("""<h2 class="section-heading" id="reproduction-guide">How can I run evaluations?</h2>""")
     gr.Markdown("""Coming soon...""")
 
-
-
-
-
 async def main():
     # Preprocess traces
     # preprocessor = TracePreprocessor()