Upload app.py

app.py

@@ -87,7 +87,7 @@ def update_task_details(benchmark_name, agent_name, task_id):
 
     summary = analysis.get('task_analysis', {})
     
-    overview = f"### Task Overview\n\n{summary.get('overview', 'No overview available.')}\n\n"
+    overview = f"### Summary\n\n{summary.get('overview', 'No overview available.')}\n\n"
     # overview += f"### Successes\n{summary.get('key_successes', 'No successes listed.')}\n\n"
     # overview += f"### Challenges\n{summary.get('main_challenges', 'No challenges listed.')}\n\n"
     # overview += f"### Overall Assessment\n{summary.get('overall_assessment', 'No assessment available.')}\n\n"
@@ -283,7 +283,7 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
 <header class="hal-header">
     <h1 class="hal-title">Holistic Agent Leaderboard (HAL)</h1>
     <p class="hal-subtitle">
-        A centralized, standardized, and cost-aware leaderboard for evaluating agents.
+        A standardized, cost-aware, and third-party leaderboard for evaluating agents.
     </p>
 </header>""")
     gr.HTML("""
@@ -332,21 +332,21 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
 
 <div class="feature-row">
     <div class="feature-column">
-        <div class="feature-keyword">Centralized</div>
+        <div class="feature-keyword">Standardized</div>
         <div class="feature-content">
             <p class="feature-description">Evaluations across agent benchmarks are all recorded to a single leaderboard that evaluates every listed agent in the same way.</p>
         </div>
     </div>
     <div class="feature-column">
-        <div class="feature-keyword">Third-party</div>
+        <div class="feature-keyword">Cost-controlled</div>
         <div class="feature-content">
-            <p class="feature-description">Agent developers clearly have competing objectives in reporting accuracy: they want to achieve state-of-the-art performance.</p>
+            <p class="feature-description">For downstream users, understanding the cost of running agents is a significant need for adoption. For agent developers, cost-controlled evaluations help develop accurate baselines.</p>
         </div>
     </div>
     <div class="feature-column">
-        <div class="feature-keyword">Cost-controlled</div>
+        <div class="feature-keyword">Third-party</div>
         <div class="feature-content">
-            <p class="feature-description">For downstream users, understanding the cost of running agents is a significant need for adoption. For agent developers, cost-controlled evaluations help develop accurate baselines.</p>
+            <p class="feature-description">Agent developers clearly have competing objectives in reporting accuracy: they want to achieve state-of-the-art performance.</p>
         </div>
     </div>
 </div>
@@ -388,8 +388,34 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
         line-height: 1.6;
         color: #333;
     }
+    .user-type-links a {
+        display: inline-block;
+        padding: 5px 12px;
+        background-color: #f0f4f8;
+        color: #2c3e50 !important; /* Force the color change */
+        text-decoration: none !important; /* Force remove underline */
+        border-radius: 15px;
+        font-size: 0.85em;
+        transition: all 0.3s ease;
+        border: 1px solid #e1e8ed;
+    }
+    .user-type-links a:hover {
+        background-color: #3498db;
+        color: white !important; /* Force the color change on hover */
+        transform: translateY(-2px);
+        box-shadow: 0 2px 5px rgba(52, 152, 219, 0.2);
+        text-decoration: none !important; /* Ensure no underline on hover */
+    }
+    .user-type-links a:visited {
+        color: #2c3e50 !important; /* Ensure visited links have the same color */
+    }
+    .user-type-links a::before {
+        content: "→";
+        margin-right: 5px;
+        font-size: 1.1em;
+    }
 </style>
-<br/>
+
 <h2 class="section-heading">Who is it for?</h2>
 <p>We see HAL being useful for four types of users:</p>
 
@@ -397,21 +423,38 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
     <div class="user-type">
         <h3 class="user-type-title">Downstream Users & Procurers</h3>
         <p class="user-type-description">Customers looking to deploy agents can get visibility into existing benchmarks, know developers building useful agents, and identify the state of the art for both cost and accuracy for their tasks of interest.</p>
+        <div class="user-type-links">
+            <a href="#leaderboards">Leaderboards</a>
+        </div>
     </div>
     <div class="user-type">
         <h3 class="user-type-title">Agent Benchmark Developers</h3>
         <p class="user-type-description">Reporting results on a centralized leaderboard could allow improved visibility into agent benchmarks that measure real-world utility.</p>
+        <div class="user-type-links">
+            <a href="#benchmark-submission">Add a Benchmark</a>
+        </div>
     </div>
     <div class="user-type">
         <h3 class="user-type-title">Agent Developers</h3>
         <p class="user-type-description">HAL allows for easy reproduction of past agents, clear comparison with past baselines, and a straightforward way to compete on a leaderboard.</p>
+        <div class="user-type-links">
+            <a href="#agent-submission">Submit an Agent</a>
+            <a href="#leaderboards">Leaderboards</a>
+            <a href="#reproduction-guide">Reproduction Guide</a>
+        </div>
     </div>
     <div class="user-type">
         <h3 class="user-type-title">Safety Researchers</h3>
         <p class="user-type-description">Understanding agent capabilities on real-world safety threats and their associated costs is crucial. For example, Cybench evaluations could provide insights into agent performance and affordability for potential adversaries.</p>
+        <div class="user-type-links">
+            <a href="#cybench-results">Cybench Leaderboard (coming soon)</a>
+            <a href="#agent-monitor">Agent Monitor</a>
+        </div>
     </div>
 </div>
-<br/>
+</br>
+<h2 class="section-heading" id="leaderboards">Leaderboards</h2>
+<p>Select a benchmark to see the agent leaderboard. Verified results have been run by the HAL team:</p>
 """)
     
     with gr.Tabs():
@@ -447,113 +490,139 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
             ),
             outputs=[task_success_heatmap]
             )
-            gr.Markdown("")
-            gr.Markdown("")
-            gr.Markdown("## Failure report for each agent")
-            gr.Markdown('Select an agent to see why the agent fails to solve tasks correctly. Note that these descriptions (and the failure categories) are generated by LLM-based evaluations of the agent logs and may contain inaccuracies.')
-            with gr.Row():
-                with gr.Column(scale=1):
-                    failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
-            with gr.Row():
-                with gr.Column(scale=1):
-                    failure_categories_overview = gr.Markdown()
-            
-                with gr.Column(scale=1):
-                    failure_categories_chart = gr.Plot()
-
-            # Initialize the failure report agent dropdown with all agents
-            demo.load(update_agent_dropdown, 
-                    inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                    outputs=[failure_report_agent_dropdown])
-            
-            # Update failure report when agent is selected
-            failure_report_agent_dropdown.change(update_failure_report,
-                                                inputs=[failure_report_agent_dropdown, gr.Textbox(value="usaco", visible=False)],
-                                                outputs=[failure_categories_overview, failure_categories_chart])
-            
-            gr.Markdown("")
-            gr.Markdown("")
-            gr.Markdown("## Agent monitor")
-            with gr.Row():
-                with gr.Column(scale=1):
-                    agent_dropdown = gr.Dropdown(label="Select Agent")
-                with gr.Column(scale=1):
-                    task_dropdown = gr.Dropdown(label="Select USACO Task")
-            with gr.Row():
-                task_overview = gr.Markdown()
-            with gr.Row():
-                flow_chart = gr.Plot(label="Task Flow")
 
-            # Initialize the agent dropdown with the best agent
-            demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
-            demo.load(update_task_analysis, inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-
-            agent_dropdown.change(update_task_analysis, 
-                                  inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown],
-                                  outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
-            task_dropdown.change(update_task_details,
-                                 inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown, task_dropdown],
-                                 outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
+            gr.HTML("""
+            <style>
+                .grouped-section {
+                    border: 2px solid #dee2e6; /* Color matching unactivated tabs */
+                    border-radius: 10px;
+                    padding: 30px;
+                    margin-top: 40px;
+                    margin-bottom: 40px;
+                    position: relative;
+                }
+
+                .grouped-section-title {
+                    font-size: 1.7em;
+                    font-weight: bold;
+                    color: #2c3e50;
+                    margin-bottom: 20px;
+                    padding-bottom: 10px;
+                    border-bottom: 2px solid #dee2e6;
+                }
+            </style>
+            """)
+            with gr.Group(elem_classes=["grouped-section"]):
+                gr.Markdown("# Agent monitor", elem_classes=["grouped-section-title"], elem_id="agent-monitor")
+
+                gr.HTML('<div style="height: 10px;"></div>')
+                gr.Markdown("## Failure report for each agent")
+                gr.Markdown('Select an agent to see why the agent fails to solve tasks correctly. Note that these descriptions (and the failure categories) are generated by LLM-based evaluations of the agent logs and may contain inaccuracies.')
+                gr.HTML('<div style="height: 10px;"></div>')
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
+                gr.HTML('<div style="height: 10px;"></div>')
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        failure_categories_overview = gr.Markdown()
+                
+                    with gr.Column(scale=1):
+                        failure_categories_chart = gr.Plot()
+
+                # Initialize the failure report agent dropdown with all agents
+                demo.load(update_agent_dropdown, 
+                        inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                        outputs=[failure_report_agent_dropdown])
+                
+                # Update failure report when agent is selected
+                failure_report_agent_dropdown.change(update_failure_report,
+                                                    inputs=[failure_report_agent_dropdown, gr.Textbox(value="usaco", visible=False)],
+                                                    outputs=[failure_categories_overview, failure_categories_chart])
+
+                gr.HTML('<div style="height: 30px;"></div>')
+                gr.Markdown("## Task overview")
+                gr.HTML('<div style="height: 10px;"></div>')
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        agent_dropdown = gr.Dropdown(label="Select Agent")
+                    with gr.Column(scale=1):
+                        task_dropdown = gr.Dropdown(label="Select USACO Task")
+                gr.HTML('<div style="height: 10px;"></div>')
+                with gr.Row():
+                    task_overview = gr.Markdown()
+                with gr.Row():
+                    flow_chart = gr.Plot(label="Task Flow")
+
+                # Initialize the agent dropdown with the best agent
+                demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
+                demo.load(update_task_analysis, inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
+
+                agent_dropdown.change(update_task_analysis, 
+                                    inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown],
+                                    outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
+                task_dropdown.change(update_task_details,
+                                    inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown, task_dropdown],
+                                    outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
             
-            gr.Markdown("")
-            gr.Markdown("")
             gr.Markdown("## Raw predictions")
-            with gr.Row():
-                with gr.Column(scale=1):
-                    raw_agent_dropdown = gr.Dropdown(label="Select Agent")
-                with gr.Column(scale=1):
-                    raw_task_dropdown = gr.Dropdown(label="Select Task")
-                with gr.Column(scale=1):
-                    raw_step_dropdown = gr.Dropdown(label="Select Step")
-            
-            with gr.Row():
-                raw_call_details = gr.HTML()
-            
-            def update_raw_task_dropdown(agent_name):
-                analyzed_traces = get_analyzed_traces(agent_name, "usaco")
-                if not analyzed_traces:
-                    return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
-                task_ids = list(analyzed_traces.keys())
-                steps = analyzed_traces[task_ids[0]]['steps']
-                return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(get_analyzed_traces(agent_name, "usaco")[task_ids[0]]['steps'][0], 0)
-
-            def update_raw_step_dropdown(agent_name, task_id):
-                analyzed_traces = get_analyzed_traces(agent_name, "usaco")
-                if not analyzed_traces or task_id not in analyzed_traces:
-                    return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
-                steps = analyzed_traces[task_id]['steps']
-                return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
-
-            def update_raw_call_details(agent_name, task_id, step_index):
-                analyzed_traces = get_analyzed_traces(agent_name, "usaco")
-                if not analyzed_traces or task_id not in analyzed_traces:
-                    return "No data available for this selection."
-                steps = analyzed_traces[task_id]['steps']
-                if step_index is None:
-                    return "Invalid step selection."
-                step = steps[step_index]
-                return format_call_info(step, step_index)
-
-            # Initialize the raw agent dropdown with all agents
-            demo.load(update_agent_dropdown, 
-                    inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
-                    outputs=[raw_agent_dropdown])
-            demo.load(update_raw_task_dropdown,
-                    inputs=[raw_agent_dropdown],
-                    outputs=[raw_task_dropdown, raw_step_dropdown])
-            demo.load(update_raw_call_details,
-                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
-                    outputs=[raw_call_details])
-
-            raw_agent_dropdown.change(update_raw_task_dropdown, 
-                                    inputs=[raw_agent_dropdown], 
-                                    outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
-            raw_task_dropdown.change(update_raw_step_dropdown, 
-                                    inputs=[raw_agent_dropdown, raw_task_dropdown], 
-                                    outputs=[raw_step_dropdown, raw_call_details])
-            raw_step_dropdown.change(update_raw_call_details, 
-                                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
-                                    outputs=[raw_call_details])
+            gr.Markdown('Select an agent to see the raw predictions made by the agent for each task. We also provide information on token usage for each call.')
+            with gr.Accordion("Expand to inspect raw predictions of agents...", open=False):
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        raw_agent_dropdown = gr.Dropdown(label="Select Agent")
+                    with gr.Column(scale=1):
+                        raw_task_dropdown = gr.Dropdown(label="Select Task")
+                    with gr.Column(scale=1):
+                        raw_step_dropdown = gr.Dropdown(label="Select Step")
+                with gr.Row():
+                    raw_call_details = gr.HTML()
+                
+                def update_raw_task_dropdown(agent_name):
+                    analyzed_traces = get_analyzed_traces(agent_name, "usaco")
+                    if not analyzed_traces:
+                        return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
+                    task_ids = list(analyzed_traces.keys())
+                    steps = analyzed_traces[task_ids[0]]['steps']
+                    return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(get_analyzed_traces(agent_name, "usaco")[task_ids[0]]['steps'][0], 0)
+
+                def update_raw_step_dropdown(agent_name, task_id):
+                    analyzed_traces = get_analyzed_traces(agent_name, "usaco")
+                    if not analyzed_traces or task_id not in analyzed_traces:
+                        return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
+                    steps = analyzed_traces[task_id]['steps']
+                    return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
+
+                def update_raw_call_details(agent_name, task_id, step_index):
+                    analyzed_traces = get_analyzed_traces(agent_name, "usaco")
+                    if not analyzed_traces or task_id not in analyzed_traces:
+                        return "No data available for this selection."
+                    steps = analyzed_traces[task_id]['steps']
+                    if step_index is None:
+                        return "Invalid step selection."
+                    step = steps[step_index]
+                    return format_call_info(step, step_index)
+
+                # Initialize the raw agent dropdown with all agents
+                demo.load(update_agent_dropdown, 
+                        inputs=[gr.Textbox(value="usaco", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                        outputs=[raw_agent_dropdown])
+                demo.load(update_raw_task_dropdown,
+                        inputs=[raw_agent_dropdown],
+                        outputs=[raw_task_dropdown, raw_step_dropdown])
+                demo.load(update_raw_call_details,
+                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
+                        outputs=[raw_call_details])
+
+                raw_agent_dropdown.change(update_raw_task_dropdown, 
+                                        inputs=[raw_agent_dropdown], 
+                                        outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
+                raw_task_dropdown.change(update_raw_step_dropdown, 
+                                        inputs=[raw_agent_dropdown, raw_task_dropdown], 
+                                        outputs=[raw_step_dropdown, raw_call_details])
+                raw_step_dropdown.change(update_raw_call_details, 
+                                        inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
+                                        outputs=[raw_call_details])
         
         
         with gr.Tab("SWE-Bench Verified"):
@@ -632,8 +701,7 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
             task_dropdown.change(update_task_details,
                                  inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown, task_dropdown],
                                  outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
-            gr.Markdown("")
-            gr.Markdown("")
+            
             gr.Markdown("## Raw predictions")
             with gr.Row():
                 with gr.Column(scale=1):
@@ -769,8 +837,7 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
                                  inputs=[gr.Textbox(value="swebench_lite", visible=False), agent_dropdown, task_dropdown],
                                  outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
 
-            gr.Markdown("")
-            gr.Markdown("")
+           
             gr.Markdown("## Raw predictions")
             with gr.Row():
                 with gr.Column(scale=1):
@@ -893,8 +960,7 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
                                  inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown, task_dropdown],
                                  outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
 
-            gr.Markdown("")
-            gr.Markdown("")
+        
             gr.Markdown("## Raw predictions")
             with gr.Row():
                 with gr.Column(scale=1):
@@ -958,6 +1024,14 @@ with gr.Blocks(theme=my_theme, css='css.css') as demo:
             gr.Markdown((Path(__file__).parent / "about.md").read_text())
 
 
+    gr.HTML("""<h2 class="section-heading" id="agent-submission">How to add an agent?</h2>
+<p>Below we provide a guide on how to add an agent to the leaderboard:</p>""")
+    gr.HTML("""<h2 class="section-heading" id="benchmark-submission">How to add a benchmark?</h2>
+<p>Below we provide a guide on how to add a benchmark to the leaderboard:</p>""")
+    gr.HTML("""<h2 class="section-heading" id="reproduction-guide">How can I run evaluations?</h2>
+<p>Below we provide a guide on how to reproduce evaluations:</p>""")
+
+