Upload 3 files

utils/db.py

@@ -6,7 +6,9 @@ from functools import lru_cache
 import threading
 import pandas as pd
 import ast
+from scipy import stats
 import yaml
+import numpy as np
 
 class TracePreprocessor:
     def __init__(self, db_path='preprocessed_traces.db'):
@@ -24,16 +26,20 @@ class TracePreprocessor:
                 CREATE TABLE IF NOT EXISTS preprocessed_traces (
                     benchmark_name TEXT,
                     agent_name TEXT,
+                    date TEXT,
+                    run_id TEXT,
                     raw_logging_results BLOB,
-                    PRIMARY KEY (benchmark_name, agent_name)
+                    PRIMARY KEY (benchmark_name, agent_name, run_id)
                 )
             ''')
             conn.execute('''
                 CREATE TABLE IF NOT EXISTS failure_reports (
                     benchmark_name TEXT,
                     agent_name TEXT,
+                    date TEXT,
+                    run_id TEXT,
                     failure_report BLOB,
-                    PRIMARY KEY (benchmark_name, agent_name)
+                    PRIMARY KEY (benchmark_name, agent_name, run_id)
                 )
             ''')
             conn.execute('''
@@ -41,6 +47,7 @@ class TracePreprocessor:
                     benchmark_name TEXT,
                     agent_name TEXT,
                     date TEXT,
+                    run_id TEXT,
                     successful_tasks TEXT,
                     failed_tasks TEXT,
                     total_cost REAL,
@@ -58,7 +65,7 @@ class TracePreprocessor:
                     amp_parkinsons_disease_progression_prediction_score REAL,
                     cifar10_score REAL,
                     imdb_score REAL,
-                    PRIMARY KEY (benchmark_name, agent_name)
+                    PRIMARY KEY (benchmark_name, agent_name, run_id)
                 )
             ''')
 
@@ -70,15 +77,17 @@ class TracePreprocessor:
                 data = json.load(f)
                 agent_name = data['config']['agent_name']
                 benchmark_name = data['config']['benchmark_name']
+                date = data['config']['date']
+                config = data['config']
 
             try:
                 raw_logging_results = pickle.dumps(data['raw_logging_results'])
                 with self.get_conn() as conn:
                     conn.execute('''
                         INSERT OR REPLACE INTO preprocessed_traces 
-                        (benchmark_name, agent_name, raw_logging_results) 
-                        VALUES (?, ?, ?)
-                    ''', (benchmark_name, agent_name, raw_logging_results))
+                        (benchmark_name, agent_name, date, run_id, raw_logging_results) 
+                        VALUES (?, ?, ?, ?, ?)
+                    ''', (benchmark_name, agent_name, date, config['run_id'], raw_logging_results))
             except Exception as e:
                 print(f"Error preprocessing raw_logging_results in {file}: {e}")
 
@@ -86,10 +95,10 @@ class TracePreprocessor:
                 failure_report = pickle.dumps(data['failure_report'])
                 with self.get_conn() as conn:
                     conn.execute('''
-                        INSERT OR REPLACE INTO failure_reports 
-                        (benchmark_name, agent_name, failure_report) 
-                        VALUES (?, ?, ?)
-                    ''', (benchmark_name, agent_name, failure_report))
+                        INSERT INTO failure_reports 
+                        (benchmark_name, agent_name, date, run_id, failure_report)
+                        VALUES (?, ?, ?, ? ,?)
+                    ''', (benchmark_name, agent_name, date, config['run_id'], failure_report))
             except Exception as e:
                 print(f"Error preprocessing failure_report in {file}: {e}")
 
@@ -98,13 +107,14 @@ class TracePreprocessor:
                 results = data['results']
                 with self.get_conn() as conn:
                     conn.execute('''
-                        INSERT OR REPLACE INTO parsed_results 
-                        (benchmark_name, agent_name, date, successful_tasks, failed_tasks, total_cost, accuracy, precision, recall, f1_score, auc, overall_score, vectorization_score, fathomnet_score, feedback_score, house_price_score, spaceship_titanic_score, amp_parkinsons_disease_progression_prediction_score, cifar10_score, imdb_score) 
-                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        INSERT INTO parsed_results 
+                        (benchmark_name, agent_name, date, run_id, successful_tasks, failed_tasks, total_cost, accuracy, precision, recall, f1_score, auc, overall_score, vectorization_score, fathomnet_score, feedback_score, house_price_score, spaceship_titanic_score, amp_parkinsons_disease_progression_prediction_score, cifar10_score, imdb_score) 
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                     ''', (
                         benchmark_name,
                         agent_name,
                         config['date'],
+                        config['run_id'],
                         str(results.get('successful_tasks')),
                         str(results.get('failed_tasks')),
                         results.get('total_cost'),
@@ -129,30 +139,62 @@ class TracePreprocessor:
     @lru_cache(maxsize=100)
     def get_analyzed_traces(self, agent_name, benchmark_name):
         with self.get_conn() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT raw_logging_results FROM preprocessed_traces 
+            query = '''
+                SELECT agent_name, raw_logging_results, date FROM preprocessed_traces 
                 WHERE benchmark_name = ? AND agent_name = ?
-            ''', (benchmark_name, agent_name))
-            result = cursor.fetchone()
-        if result:
-            return pickle.loads(result[0])
-        return None
+            '''
+            df = pd.read_sql_query(query, conn, params=(benchmark_name, agent_name))
+
+
+        # check for each row if raw_logging_results is not None with pickle.loads because it is stored as a byte string
+        df = df[df['raw_logging_results'].apply(lambda x: pickle.loads(x) is not None and x != 'None')]
+
+        if len(df) == 0:
+            return None
+
+        # select latest run
+        df = df.sort_values('date', ascending=False).groupby('agent_name').first().reset_index()
+
+
+        return pickle.loads(df['raw_logging_results'][0])
+
 
     @lru_cache(maxsize=100)
     def get_failure_report(self, agent_name, benchmark_name):
         with self.get_conn() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT failure_report FROM failure_reports 
+            query = '''
+                SELECT agent_name, date, failure_report FROM failure_reports 
                 WHERE benchmark_name = ? AND agent_name = ?
-            ''', (benchmark_name, agent_name))
-            result = cursor.fetchone()
-        if result:
-            return pickle.loads(result[0])
-        return None
+            '''
+            df = pd.read_sql_query(query, conn, params=(benchmark_name, agent_name))
+
+        # Select only rows for which failure report is not None and None is a string
+        df = df[df['failure_report'].apply(lambda x: pickle.loads(x) is not None and x != 'None')]
+
+        if len(df) == 0:
+            return None
+        
+
+        # if there is multiple failure reports, take the last one
+        df = df.sort_values('date', ascending=False).groupby('agent_name').first().reset_index()
+
+        # if there is a failure report, return the first one
+        return pickle.loads(df['failure_report'][0])
+    
+    def _calculate_ci(self, data, confidence=0.95):
+        data = data[np.isfinite(data)]
+
+        if len(data) < 2:
+            return '', '', '' # No CI for less than 2 samples
+        n = len(data)
+
+        mean = np.mean(data)
+        sem = stats.sem(data)
+
+        ci = stats.t.interval(confidence, n-1, loc=mean, scale=sem)
+        return mean, ci[0], ci[1]
     
-    def get_parsed_results(self, benchmark_name):
+    def get_parsed_results(self, benchmark_name, aggregate=True):
         with self.get_conn() as conn:
             query = '''
                 SELECT * FROM parsed_results 
@@ -167,13 +209,65 @@ class TracePreprocessor:
         # Add 'Verified' column
         df['Verified'] = df.apply(lambda row: '✓' if (benchmark_name, row['agent_name']) in verified_agents else '', axis=1)
 
+
+        
+        # Add column for how many times an agent_name appears in the DataFrame
+        df['Runs'] = df.groupby('agent_name')['agent_name'].transform('count')
+
+        # Compute the 95% confidence interval for accuracy and cost for agents that have been run more than once
+        df['acc_ci'] = None
+        df['cost_ci'] = None
+
+        for agent_name in df['agent_name'].unique():
+            agent_df = df[df['agent_name'] == agent_name]
+            
+            if len(agent_df) > 1:
+                accuracy_mean, accuracy_lower, accuracy_upper = self._calculate_ci(agent_df['accuracy'])
+                cost_mean, cost_lower, cost_upper = self._calculate_ci(agent_df['total_cost'])
+                
+                # format the confidence interval with +/- sign
+                accuracy_ci = f"± {abs(accuracy_mean - accuracy_lower):.3f}"
+                cost_ci = f"± {abs(cost_mean - cost_lower):.3f}"
+                
+                df.loc[df['agent_name'] == agent_name, 'acc_ci'] = accuracy_ci
+                df.loc[df['agent_name'] == agent_name, 'cost_ci'] = cost_ci
+
+
+        df = df.drop(columns=['successful_tasks', 'failed_tasks', 'run_id'], axis=1)
+
+        if aggregate:
+            # For agents that have been run more than once, compute the average accuracy and cost and use that as the value in the DataFrame
+            df = df.groupby('agent_name').agg({
+                'date': 'first',
+                'total_cost': 'mean',
+                'accuracy': 'mean',
+                'precision': 'mean',
+                'recall': 'mean',
+                'f1_score': 'mean',
+                'auc': 'mean',
+                'overall_score': 'mean',
+                'vectorization_score': 'mean',
+                'fathomnet_score': 'mean',
+                'feedback_score': 'mean',
+                'house_price_score': 'mean',
+                'spaceship_titanic_score': 'mean',
+                'amp_parkinsons_disease_progression_prediction_score': 'mean',
+                'cifar10_score': 'mean',
+                'imdb_score': 'mean',
+                'Verified': 'first',
+                'Runs': 'first',
+                'acc_ci': 'first',
+                'cost_ci': 'first'
+            }).reset_index()
+
         # Round float columns to 3 decimal places
         float_columns = ['total_cost', 'accuracy', 'precision', 'recall', 'f1_score', 'auc', 'overall_score', 'vectorization_score', 'fathomnet_score', 'feedback_score', 'house-price_score', 'spaceship-titanic_score', 'amp-parkinsons-disease-progression-prediction_score', 'cifar10_score', 'imdb_score']
         for column in float_columns:
             if column in df.columns:
                 df[column] = df[column].round(3)
 
-        df = df.drop(columns=['successful_tasks', 'failed_tasks'], axis=1)
+        # sort by accuracy
+        df = df.sort_values('accuracy', ascending=False)
 
         # Rename columns
         df = df.rename(columns={
@@ -193,7 +287,9 @@ class TracePreprocessor:
             'spaceship_titanic_score': 'Spaceship Titanic Score',
             'amp_parkinsons_disease_progression_prediction_score': 'AMP Parkinsons Disease Progression Prediction Score',
             'cifar10_score': 'CIFAR10 Score',
-            'imdb_score': 'IMDB Score'
+            'imdb_score': 'IMDB Score',
+            'acc_ci': 'Accuracy CI',
+            'cost_ci': 'Total Cost CI'
         })
 
         return df
@@ -201,11 +297,14 @@ class TracePreprocessor:
     def get_task_success_data(self, benchmark_name):
         with self.get_conn() as conn:
             query = '''
-                SELECT agent_name, successful_tasks, failed_tasks
+                SELECT agent_name, accuracy, successful_tasks, failed_tasks
                 FROM parsed_results 
                 WHERE benchmark_name = ?
             '''
             df = pd.read_sql_query(query, conn, params=(benchmark_name,))
+
+        # for agent_names that have been run more than once, take the run with the highest accuracy
+        df = df.sort_values('accuracy', ascending=False).groupby('agent_name').first().reset_index()
         
         # Get all unique task IDs
         task_ids = set()

utils/processing.py

@@ -10,6 +10,7 @@ import aiosmtplib
 from agent_monitor.monitor import analyze_agent_steps
 from agent_monitor.failure_report import analyze_agent_performance, AsyncOpenAIClient
 import traceback
+from tqdm import tqdm
 
 async def check_and_process_uploads():
     upload_dir =  "evals_upload"
@@ -58,12 +59,14 @@ async def check_and_process_uploads():
 
     print(f"Processing {len(unprocessed_uploads)} new uploads.")
     tasks = []
-    for upload in unprocessed_uploads:
+    for upload in tqdm(unprocessed_uploads):
         upload_path = os.path.join(upload_dir, upload)
         processed_path = os.path.join(processed_dir, upload)
-        tasks.append(process_single_upload(upload_path, processed_path))
-    
-    await asyncio.gather(*tasks)
+        # tasks.append(process_single_upload(upload_path, processed_path)) # for async processing
+        await process_single_upload(upload_path, processed_path)
+
+
+    # await asyncio.gather(*tasks) # for async processing
 
 
 async def process_single_upload(upload_path, processed_path):
@@ -77,12 +80,9 @@ async def process_single_upload(upload_path, processed_path):
         # Move the file to processed directory
         # await asyncio.to_thread(shutil.move, upload_path, processed_path)
         
-        # Send email notification
-        # await send_email_notification(upload_path.name, check_result, "Processing successful")
     else:
         print(f"Upload check failed for {upload_path}: {check_result['message']}")
-        # Send email notification about the failed check
-        # await send_email_notification(upload_path.name, check_result, "Upload check failed")
+
         
         
 async def check_upload_structure(file_path):
@@ -123,11 +123,11 @@ async def process_upload(input_path, output_path):
     openai_client = AsyncOpenAIClient(model="gpt-4o-mini")
 
     try:
-        processed_calls = await analyze_agent_steps(data['raw_logging_results'], openai_client, llm_eval=False)
-        data['raw_logging_results'] = processed_calls
+        processed_calls = await analyze_agent_steps(data['raw_logging_results'], openai_client, llm_eval=True)
+        failure_report = await analyze_agent_performance(data['raw_logging_results'], data['results']['failed_tasks'], openai_client)
 
-        # failure_report = await analyze_agent_performance(data['raw_logging_results'], data['results']['failed_tasks'], openai_client)
-        # data['failure_report'] = None
+        data['raw_logging_results'] = processed_calls
+        data['failure_report'] = failure_report
     except Exception as e:
         traceback.print_exc()
         print(f"Error in processing: {str(e)}")
@@ -163,4 +163,5 @@ async def send_email_notification(filename, check_result, status):
         use_tls=True,
         username=sender_email,
         password=password
-    )
+    )
+

utils/viz.py

@@ -3,8 +3,29 @@ import plotly.express as px
 from utils.pareto import Agent, compute_pareto_frontier
 import plotly.graph_objects as go
 import textwrap
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2
+from scipy import stats
+
+
+def create_leaderboard(df, ci_metrics = None):
+    # cast dtypes to string
+    df = df.astype(str)
+
+    # for each metric join metric and metric CI columns
+    if ci_metrics:
+        for metric in ci_metrics:
+            CI_metric = metric + ' CI'
+            # for rows in the df for which CI metric is not None, join the metric and CI columns by looping through the CI metrics columns
+            for i, row in df.iterrows():
+                if str(row[CI_metric]) != 'None':
+                    df.at[i, metric] = str(row[metric]) + " (" + str(row[CI_metric]) + ")"
+
+    return df
 
 def create_task_success_heatmap(df, benchmark_name):
+
     # Calculate agent accuracy
     agent_accuracy = df.groupby('Agent Name')['Success'].mean().sort_values(ascending=False)
     
@@ -17,16 +38,30 @@ def create_task_success_heatmap(df, benchmark_name):
     # Sort the pivot table
     pivot_df = pivot_df.reindex(index=agent_accuracy.index, columns=task_success_rate.index)
 
+    # Calculate tasks solved across all agents
+    tasks_solved = (pivot_df.sum(axis=0) > 0).astype(int)
+    # Total number of tasks (columns)
+    total_tasks = len(pivot_df.columns)
+    
+    # Add the new row to the pivot table
+    tasks_solved_df = pd.DataFrame(tasks_solved).T
+    tasks_solved_df.index = [f'<b>Tasks Solved: {tasks_solved.sum()}/{total_tasks} (All Agents)</b>']
+    # print number of tasks solved
+    pivot_df = pd.concat([pivot_df, tasks_solved_df])
+
     num_agents = len(pivot_df.index)
     row_height = 30  # Fixed height for each row in pixels
     total_height = num_agents * row_height
     
+    # Create a custom colorscale
+    colorscale=[[0, 'white'], [1, '#3498db']]
+
     # Create the heatmap
     fig = go.Figure(data=go.Heatmap(
         z=pivot_df.values,
         y=pivot_df.index,
         x=pivot_df.columns,
-        colorscale=[[0, 'white'], [1, '#3498db']],  # White for failed, green for success
+        colorscale=colorscale,
         showscale=False,
         hovertemplate='<b>Agent:</b> %{y}<br>' +
                       '<b>Task:</b> %{x}<br>' +
@@ -36,18 +71,17 @@ def create_task_success_heatmap(df, benchmark_name):
     # Update the layout
     fig.update_layout(
         xaxis_title='Task ID',
-        height=total_height,
-        # width=1150,
+        height=total_height + 50,  # Add extra space for the new row
         yaxis=dict(
             autorange='reversed',
-            showticklabels=True,  # Show y-axis tick labels (agent names)
+            showticklabels=True,
             showline=True,
             linecolor='black',
             showgrid=False
         ),
         xaxis=dict(
             side='top',
-            showticklabels=False,  # Hide x-axis tick labels (task IDs)
+            showticklabels=False,
             showline=True,
             linecolor='black',
             showgrid=False
@@ -136,65 +170,173 @@ def create_bar_chart(categories, values, x_label, y_label, title):
     return fig
 
 def create_scatter_plot(df, x: str, y: str, x_label: str = None, y_label: str = None, hover_data: list = None):
-    agents = [Agent(row['Total Cost'], row['Accuracy']) for i, row in df.iterrows()]
+    # agents = [Agent(row['Total Cost'], row['Accuracy']) for i, row in df.iterrows()]
+    # instead of creating one Agent object for each row, we can create one Agent object for each unique agent and use the mean of the cost and accuracy values
+    unique_agents = df['Agent Name'].unique()
+    agents = [Agent(df[df['Agent Name'] == agent]['Total Cost'].mean(), df[df['Agent Name'] == agent]['Accuracy'].mean()) for agent in unique_agents]
+
     pareto_frontier = compute_pareto_frontier(agents)
 
-    fig = px.scatter(df, 
-                     x=x, 
-                     y=y,
-                     custom_data=hover_data)
-    fig.update_traces(
-            hovertemplate="<br>".join([
-                "<b>Agent</b>: %{customdata[0]}",
-                "<b>Total Cost</b>: $%{x:.1f}",
-                "<b>Accuracy</b>: %{y:.1%}",
-            ])
-        )
-    
-    fig.update_traces(marker=dict(size=10, color='#3498db'),
-                      hoverlabel=dict(bgcolor="white", font_size=12, font_family="Arial"),)
-    
+    fig = go.Figure()
 
     # Sort the Pareto frontier points by x-coordinate
     pareto_points = sorted([(agent.total_cost, agent.accuracy) for agent in pareto_frontier], key=lambda x: x[0])
-    
     # Add the Pareto frontier line
     fig.add_trace(go.Scatter(
         x=[point[0] for point in pareto_points],
         y=[point[1] for point in pareto_points],
         mode='lines',
         name='Pareto Frontier',
+        hoverinfo=None,
         line=dict(color='black', width=1, dash='dash')
     ))
 
-    fig.update_yaxes(rangemode="tozero")
-    fig.update_xaxes(rangemode="tozero")
+    # Plot scatter points and error bars for each agent
+    unique_agents = df[hover_data[0]].unique()
+    for agent in unique_agents:
+        agent_data = df[df[hover_data[0]] == agent]
+
+        x_value = [np.mean(agent_data[x].values)]
+        y_value = [np.mean(agent_data[y].values)]
+
+        if len(agent_data) > 1:
+            # Calculate 95% confidence intervals
+            ci_x = stats.t.interval(0.95, len(agent_data[x])-1, loc=np.mean(agent_data[x]), scale=stats.sem(agent_data[x]))
+            ci_y = stats.t.interval(0.95, len(agent_data[y])-1, loc=np.mean(agent_data[y]), scale=stats.sem(agent_data[y]))
+
+            # # Add error bars for x (cost)
+            # fig.add_trace(go.Scatter(
+            #     x=x_value,
+            #     y=y_value,
+            #     error_x=dict(
+            #         type='data',
+            #         symmetric=False,
+            #         array=[ci_x[1] - x_value],
+            #         arrayminus=[x_value - ci_x[0]],
+            #         color='red',
+            #     ),
+            #     mode='markers',
+            #     marker=dict(color='rgba(0,0,0,0)'),
+            #     showlegend=False,
+            #     hoverinfo='none'
+            # ))
+
+            # # Add error bars for y (accuracy)
+            # fig.add_trace(go.Scatter(
+            #     x=x_value,
+            #     y=y_value,
+            #     error_y=dict(
+            #         type='data',
+            #         symmetric=False,
+            #         array=[ci_y[1] - y_value],
+            #         arrayminus=[y_value - ci_y[0]],
+            #         color='green',
+            #     ),
+            #     mode='markers',
+            #     marker=dict(color='rgba(0,0,0,0)'),
+            #     showlegend=False,
+            #     hoverinfo='none'
+            # ))
+
+            # Add error bars for x (cost minmax)
+            fig.add_trace(go.Scatter(
+                x=x_value,
+                y=y_value,
+                error_x=dict(
+                    type='data',
+                    symmetric=False,
+                    array=[np.max(agent_data[x]) - x_value],
+                    arrayminus=[x_value - np.min(agent_data[x])],
+                    color='#fec44f',
+                ),
+                mode='markers',
+                marker=dict(color='rgba(0,0,0,0)', opacity=0),
+                showlegend=False,
+                hoverinfo=None
+            ))
+
+            # Add error bars for y (accuracy minmax)
+            fig.add_trace(go.Scatter(
+                x=x_value,
+                y=y_value,
+                error_y=dict(
+                    type='data',
+                    symmetric=False,
+                    array=[np.max(agent_data[y]) - y_value],
+                    arrayminus=[y_value - np.min(agent_data[y])],
+                    color='#bdbdbd',
+                ),
+                mode='markers',
+                marker=dict(color='rgba(0,0,0,0)', opacity=0),
+                showlegend=False,
+                hoverinfo=None
+            ))
+
+        # Add scatter points for this agent
+        fig.add_trace(go.Scatter(
+            x=x_value,
+            y=y_value,
+            mode='markers',
+            marker=dict(size=10, color='#3498db'),
+            customdata=agent_data[hover_data],
+            showlegend=False,
+            hovertemplate="<br>".join([
+                "<b>Agent</b>: %{customdata[0]}",
+                "<b>Total Cost</b>: $%{x:.1f}",
+                "<b>Accuracy</b>: %{y:.1%}<extra></extra>",
+            ]),
+            hoverlabel=dict(bgcolor="white", font_size=12, font_family="Arial"),
+        ))
+
+
+
+    # Add legend entries for error bars
+    # fig.add_trace(go.Scatter(
+    #     x=[None], y=[None], mode='markers',
+    #     marker=dict(color='red', size=10),
+    #     name='Cost CI (95%)'
+    # ))
+    # fig.add_trace(go.Scatter(
+    #     x=[None], y=[None], mode='markers',
+    #     marker=dict(color='green', size=10),
+    #     name='Accuracy CI (95%)'
+    # ))
+
+    # Add legend entries for error bars
+    fig.add_trace(go.Scatter(
+        x=[None], y=[None], mode='markers',
+        marker=dict(color='#fec44f', size=10),
+        name='Cost CI (Min-Max)'
+    ))
+    fig.add_trace(go.Scatter(
+        x=[None], y=[None], mode='markers',
+        marker=dict(color='#bdbdbd', size=10),
+        name='Accuracy CI (Min-Max)'
+    ))
 
     fig.update_layout(
-    # width = 1150,
-    height = 600,
-    xaxis_title = x_label,
-    yaxis_title = y_label,
-    xaxis = dict(
-        showline = True,
-        linecolor = 'black',
-        showgrid = False),
-    yaxis = dict(
-        showline = True,
-        showgrid = False,
-        linecolor = 'black'),
-    plot_bgcolor = 'white',
-    # Legend positioning
-    legend=dict(
-        yanchor="bottom",
-        y=0.01,
-        xanchor="right",
-        x=0.98,
-        bgcolor="rgba(255, 255, 255, 0.5)"  # semi-transparent white background
+        height = 600,
+        xaxis_title = x_label,
+        yaxis_title = y_label,
+        xaxis = dict(
+            showline = True,
+            linecolor = 'black',
+            showgrid = False),
+        yaxis = dict(
+            showline = True,
+            showgrid = False,
+            linecolor = 'black'),
+        plot_bgcolor = 'white',
+        legend=dict(
+            yanchor="bottom",
+            y=0.01,
+            xanchor="right",
+            x=0.98,
+            bgcolor="rgba(255, 255, 255, 0.5)"  # semi-transparent white background
         ),
-    modebar=dict(
+        modebar=dict(
             activecolor='#1f77b4',  # Color of active tool
-            orientation='h',  # Vertical orientation
+            orientation='h',  # Horizontal orientation
             bgcolor='rgba(255,255,255,0.8)',  # Slightly transparent white background
             color='#777',  # Color of inactive tools
             add = ['pan2d'],
@@ -211,9 +353,213 @@ def create_scatter_plot(df, x: str, y: str, x_label: str = None, y_label: str =
                 'select2d',
                 'select']
         ),
-    dragmode='pan'
+        dragmode='pan'
     )
+
+    fig.update_yaxes(rangemode="tozero")
+    fig.update_xaxes(rangemode="tozero")
+
     return fig
+# def create_scatter_plot(df, x: str, y: str, x_label: str = None, y_label: str = None, hover_data: list = None):
+#     agents = [Agent(row['Total Cost'], row['Accuracy']) for i, row in df.iterrows()]
+#     pareto_frontier = compute_pareto_frontier(agents)
+
+#     fig = go.Figure()
+
+#     # Function to generate points for error ellipse
+#     def error_ellipse(x_center, y_center, x_radius, y_radius, angle, n=50):
+#         t = np.linspace(0, 2*np.pi, n)
+#         x = x_radius * np.cos(t)
+#         y = y_radius * np.sin(t)
+#         rotation = np.array([[np.cos(angle), -np.sin(angle)],
+#                              [np.sin(angle), np.cos(angle)]])
+#         xy = np.dot(rotation, np.array([x, y]))
+#         return x_center + xy[0], y_center + xy[1]
+
+#     # Create a color map for agents
+#     unique_agents = df['Agent Name'].unique()
+#     colors = px.colors.qualitative.Plotly
+#     color_map = {agent: colors[i % len(colors)] for i, agent in enumerate(unique_agents)}
+
+#     # Add scatter points and error ellipses for each agent
+#     for agent in unique_agents:
+#         agent_data = df[df['Agent Name'] == agent]
+        
+#         # Add scatter points
+#         fig.add_trace(go.Scatter(
+#             x=agent_data[x],
+#             y=agent_data[y],
+#             mode='markers',
+#             name=agent,
+#             marker=dict(size=10, color=color_map[agent]),
+#             customdata=agent_data[hover_data] if hover_data else None,
+#             hovertemplate="<br>".join([
+#                 f"<b>Agent</b>: {agent}",
+#                 f"<b>{x}</b>: ${{x:.1f}}",
+#                 f"<b>{y}</b>: {{y:.1%}}",
+#             ] + ([f"<b>{col}</b>: {{customdata[{i}]}}" for i, col in enumerate(hover_data)] if hover_data else []))
+#         ))
+
+#         # Calculate mean and standard deviation for x and y
+#         x_mean = agent_data[x].mean()
+#         y_mean = agent_data[y].mean()
+#         x_std = agent_data[x].std()
+#         y_std = agent_data[y].std()
+
+#         # Calculate correlation coefficient
+#         corr = agent_data[x].corr(agent_data[y])
+
+#         # Add error ellipses (1 and 2 standard deviations)
+#         for n_std, opacity in [(1, 0.5), (2, 0.5)]:
+#             chi2_val = chi2.ppf(0.68 if n_std == 1 else 0.95, 2)
+#             x_radius = np.sqrt(chi2_val) * x_std
+#             y_radius = np.sqrt(chi2_val) * y_std
+#             angle = np.arctan2(y_std * corr, x_std)
+            
+#             ellipse_x, ellipse_y = error_ellipse(x_mean, y_mean, x_radius, y_radius, angle)
+            
+#             fig.add_shape(type="path",
+#                           path=f"M {ellipse_x[0]}, {ellipse_y[0]} " + 
+#                                " ".join([f"L{x},{y}" for x, y in zip(ellipse_x[1:], ellipse_y[1:])]) +
+#                                " Z",
+#                           line_color=color_map[agent],
+#                           line_width=2,
+#                           opacity=opacity,
+#                           layer="below")
+
+#     # Sort the Pareto frontier points by x-coordinate
+#     pareto_points = sorted([(agent.total_cost, agent.accuracy) for agent in pareto_frontier], key=lambda x: x[0])
+    
+#     # Add the Pareto frontier line
+#     fig.add_trace(go.Scatter(
+#         x=[point[0] for point in pareto_points],
+#         y=[point[1] for point in pareto_points],
+#         mode='lines',
+#         name='Pareto Frontier',
+#         line=dict(color='black', width=1, dash='dash')
+#     ))
+
+#     fig.update_layout(
+#         height = 600,
+#         xaxis_title = x_label,
+#         yaxis_title = y_label,
+#         xaxis = dict(
+#             showline = True,
+#             linecolor = 'black',
+#             showgrid = False),
+#         yaxis = dict(
+#             showline = True,
+#             showgrid = False,
+#             linecolor = 'black'),
+#         plot_bgcolor = 'white',
+#         legend=dict(
+#             yanchor="bottom",
+#             y=0.01,
+#             xanchor="right",
+#             x=0.98,
+#             bgcolor="rgba(255, 255, 255, 0.5)"
+#         ),
+#         modebar=dict(
+#             activecolor='#1f77b4',
+#             orientation='h',
+#             bgcolor='rgba(255,255,255,0.8)',
+#             color='#777',
+#             add = ['pan2d'],
+#             remove = [
+#                 'zoom2d', 'zoomIn2d', 'zoomOut2d', 'resetScale2d',
+#                 'hoverClosestCartesian', 'hoverCompareCartesian',
+#                 'toggleSpikelines', 'lasso2d', 'lasso',
+#                 'select2d', 'select'
+#             ]
+#         ),
+#         dragmode='pan'
+#     )
+
+#     fig.update_yaxes(rangemode="tozero")
+#     fig.update_xaxes(rangemode="tozero")
+
+#     return fig
+
+# def create_scatter_plot(df, x: str, y: str, x_label: str = None, y_label: str = None, hover_data: list = None):
+#     agents = [Agent(row['Total Cost'], row['Accuracy']) for i, row in df.iterrows()]
+#     pareto_frontier = compute_pareto_frontier(agents)
+
+#     fig = px.scatter(df, 
+#                      x=x, 
+#                      y=y,
+#                      custom_data=hover_data)
+    # fig.update_traces(
+    #         hovertemplate="<br>".join([
+    #             "<b>Agent</b>: %{customdata[0]}",
+    #             "<b>Total Cost</b>: $%{x:.1f}",
+    #             "<b>Accuracy</b>: %{y:.1%}",
+    #         ])
+    #     )
+    
+#     fig.update_traces(marker=dict(size=10, color='#3498db'),
+#                       hoverlabel=dict(bgcolor="white", font_size=12, font_family="Arial"),)
+    
+
+#     # Sort the Pareto frontier points by x-coordinate
+#     pareto_points = sorted([(agent.total_cost, agent.accuracy) for agent in pareto_frontier], key=lambda x: x[0])
+    
+#     # Add the Pareto frontier line
+#     fig.add_trace(go.Scatter(
+#         x=[point[0] for point in pareto_points],
+#         y=[point[1] for point in pareto_points],
+#         mode='lines',
+#         name='Pareto Frontier',
+#         line=dict(color='black', width=1, dash='dash')
+#     ))
+
+#     fig.update_layout(
+#     # width = 1150,
+#     height = 600,
+#     xaxis_title = x_label,
+#     yaxis_title = y_label,
+#     xaxis = dict(
+#         showline = True,
+#         linecolor = 'black',
+#         showgrid = False),
+#     yaxis = dict(
+#         showline = True,
+#         showgrid = False,
+#         linecolor = 'black'),
+#     plot_bgcolor = 'white',
+#     # Legend positioning
+#     legend=dict(
+#         yanchor="bottom",
+#         y=0.01,
+#         xanchor="right",
+#         x=0.98,
+#         bgcolor="rgba(255, 255, 255, 0.5)"  # semi-transparent white background
+#         ),
+#     modebar=dict(
+#             activecolor='#1f77b4',  # Color of active tool
+#             orientation='h',  # Vertical orientation
+#             bgcolor='rgba(255,255,255,0.8)',  # Slightly transparent white background
+#             color='#777',  # Color of inactive tools
+#             add = ['pan2d'],
+#             remove = [
+#                 'zoom2d',
+#                 'zoomIn2d',
+#                 'zoomOut2d', 
+#                 'resetScale2d',
+#                 'hoverClosestCartesian', 
+#                 'hoverCompareCartesian',
+#                 'toggleSpikelines',
+#                 'lasso2d',
+#                 'lasso',
+#                 'select2d',
+#                 'select']
+#         ),
+#     dragmode='pan'
+#     )
+
+#     fig.update_yaxes(rangemode="tozero")
+#     fig.update_xaxes(rangemode="tozero")
+
+#     return fig
 
 
 import plotly.graph_objects as go