Upload 46 files

embeddings_cache.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a37a15611f94584c735999d88868417ddcd3e6865b1f01413b6f644e1e8b6ca1
+size 2803614

projects/default.json

@@ -0,0 +1,164 @@
+{
+    "project_name": "default",
+    "assertions": {
+        "deterministic": [
+            {
+                "check_type": "contains",
+                "value": "bmc"
+            },
+            {
+                "check_type": "regex",
+                "value": ".*"
+            }
+        ],
+        "misc": [],
+        "factual": false,
+        "sql-only": true,
+        "json-only": true
+    },
+    "log_history": [],
+    "accuracy_history": {
+        "DSA": [
+            [
+                "2025-02-23 23:08:34",
+                0.5624579697847366
+            ],
+            [
+                "2025-02-23 23:09:32",
+                0.5233337700366973
+            ],
+            [
+                "2025-02-23 23:14:15",
+                0.603324833241376
+            ],
+            [
+                "2025-02-23 23:15:34",
+                0.5898826479911804
+            ],
+            [
+                "2025-02-23 23:20:53",
+                0.5855140775442124
+            ],
+            [
+                "2025-02-23 23:23:21",
+                0.5792156517505646
+            ],
+            [
+                "2025-02-23 23:24:05",
+                0.6172547936439514
+            ],
+            [
+                "2025-02-23 23:26:20",
+                0.6511377811431884
+            ],
+            [
+                "2025-02-24 10:44:12",
+                0.5467980474233627
+            ],
+            [
+                "2025-02-24 14:29:43",
+                0.5033589959144592
+            ]
+        ],
+        "Technical": [
+            [
+                "2025-02-23 23:33:09",
+                50.0
+            ],
+            [
+                "2025-02-23 23:33:54",
+                60.0
+            ],
+            [
+                "2025-02-23 23:34:14",
+                60.0
+            ],
+            [
+                "2025-02-23 23:34:37",
+                45.45454545454545
+            ],
+            [
+                "2025-02-23 23:36:38",
+                70.0
+            ],
+            [
+                "2025-02-23 23:37:39",
+                54.54545454545454
+            ],
+            [
+                "2025-02-23 23:38:03",
+                50.0
+            ],
+            [
+                "2025-02-24 14:31:55",
+                90.9090909090909
+            ]
+        ],
+        "Behaviour": [
+            [
+                "2025-02-24 11:09:03",
+                0.8181818181818182
+            ],
+            [
+                "2025-02-24 11:29:39",
+                0.0
+            ],
+            [
+                "2025-02-24 11:30:28",
+                0.0
+            ],
+            [
+                "2025-02-24 11:31:35",
+                0.8181818181818182
+            ],
+            [
+                "2025-02-24 11:34:03",
+                0.5833333333333334
+            ],
+            [
+                "2025-02-24 11:38:26",
+                0.9
+            ],
+            [
+                "2025-02-24 12:08:28",
+                0.0
+            ],
+            [
+                "2025-02-24 12:11:57",
+                0.0
+            ],
+            [
+                "2025-02-24 12:13:39",
+                0.0
+            ],
+            [
+                "2025-02-24 12:16:24",
+                0.0
+            ],
+            [
+                "2025-02-24 12:19:08",
+                0.6363636363636364
+            ],
+            [
+                "2025-02-24 12:22:44",
+                0.5454545454545454
+            ],
+            [
+                "2025-02-24 12:34:16",
+                0.7272727272727273
+            ],
+            [
+                "2025-02-24 12:37:06",
+                0.9166666666666666
+            ],
+            [
+                "2025-02-24 12:39:09",
+                0.4
+            ],
+            [
+                "2025-02-24 12:50:24",
+                1.0
+            ]
+        ]
+    }
+}

projects/default_project.json

@@ -0,0 +1,13 @@
+{
+    "project_name": "default_project",
+    "assertions": {
+        "deterministic": [],
+        "misc": [],
+        "factual": false,
+        "sql-only": false,
+        "json-only": false
+        
+    },
+    "log_history": [],
+    "accuracy_history": []
+}

projects/po.json

@@ -0,0 +1,21 @@
+{
+    "project_name": "po",
+    "assertions": {
+        "deterministic": [],
+        "misc": [],
+        "factual": "",
+        "sql-only": false,
+        "json-only": false
+    },
+    "log_history": [],
+    "accuracy_history": {
+        "DSA": [],
+        "Technical": [],
+        "Behaviour": [
+            [
+                "2025-02-24 11:51:54",
+                0.7272727272727273
+            ]
+        ]
+    }
+}

requirements.txt

@@ -0,0 +1,13 @@
+streamlit
+python-docx
+PyPDF2
+groq
+python-dotenv
+nltk
+pandas
+scikit-learn==1.3.0
+matplotlib==3.7.1
+sentence-transformers==2.2.2
+rake-nltk==1.0.6
+spacy
+textblob

src/modules/module1_question_generation/.env

@@ -0,0 +1,2 @@
+# GROQ_API_KEY="xai-eVZLU4OIrvTFco272DRuIyI1EoSd54eWjsKVZ4PrepUJ8WxZOwbfDnKbsoHSd96r9npTvsYdbtUzqd6x"
+GROQ_API_KEY="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"

src/modules/module1_question_generation/app.py

@@ -0,0 +1,237 @@
+import streamlit as st
+import os
+import sys
+import numpy as np
+import matplotlib.pyplot as plt
+import datetime
+import pandas as pd
+
+# Adjust the system path to find project modules
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))
+sys.path.append(project_root)
+
+from src.modules.module2_relevancy.relevance_analyzer import EnhancedRelevanceAnalyzer
+from groq_client import GroqClient
+from file_processing import extract_text_from_file
+from src.modules.module3_compare.model import QuestionSimilarityModel
+from src.modules.module4_bias.bias import screen_questions
+from src.modules.module1_question_generation.project_controller import Project
+from src.modules.module1_question_generation.tool_controller import *
+DATASET_DIR = "dataset"
+project_control = Project()
+if 'page' not in st.session_state:
+    st.session_state.page = 'main'
+if ('accuracy_history' not in st.session_state):
+    st.session_state['accuracy_history'] = {
+                "DSA" : [],
+                "Technical" : [],
+                "Behaviour": []
+            }
+
+def main():
+    
+    if st.session_state.page == 'main':
+        sidebar()
+        if ('current_project' in st.session_state):
+            if (st.session_state['current_project']['project_name'] == 'default'):
+                st.title("Interview Question Generator & Analyzer")
+            main_page()
+        else:
+            st.subheader('No project selected')
+    elif st.session_state.page == 'configure':
+        configure_page()
+
+def sidebar():
+    st.sidebar.title("Project Options")
+    project_action = st.sidebar.selectbox("Select Action", ["Open Existing Project", "Create New Project"])
+    if project_action == "Create New Project":
+        new_project_name = st.sidebar.text_input("Enter Project Name")
+        print('Title: ', new_project_name)
+        if st.sidebar.button("Create Project") and new_project_name:
+            if new_project_name in project_control.list_projects():
+                st.sidebar.error("Project with this name already exists.")
+            else:
+                project_data = project_control.initialize_project(new_project_name)
+                st.session_state["current_project"] = project_data
+                st.success(f"Project '{new_project_name}' created successfully!")
+
+    elif project_action == "Open Existing Project":
+        existing_projects = project_control.list_projects()
+        selected_project = st.sidebar.selectbox("Select Project", existing_projects)
+        if st.sidebar.button("Open Project") and selected_project:
+            project_data = project_control.load_project(selected_project)
+            if project_data:
+                st.session_state["current_project"] = project_data
+            else:
+                st.sidebar.error("Failed to load project_control.")
+    if ('current_project' in st.session_state and  st.sidebar.button('Configure Project')):
+        st.session_state.page = 'configure'
+
+def main_page():
+    client = GroqClient()
+    analyzer = EnhancedRelevanceAnalyzer()
+    similarity_model = QuestionSimilarityModel('dataset/leetcode_dataset.csv')
+    project = st.session_state["current_project"]
+    
+    st.subheader('Project: ', project['project_name'])
+
+    job_role = st.text_input("Enter Job Role")
+    question_type = st.selectbox("Type of questions", ["DSA", "Technical", "Behaviour"])
+    jd_file = st.file_uploader("Upload Job Description (PDF/DOCX)", type=["pdf", "docx"])
+    
+
+    if jd_file and job_role and question_type and st.button('Get questions') :
+        with st.spinner("Analyzing Job Description..."):
+            jd_text = extract_text_from_file(jd_file)
+
+            if not analyzer.check_title_jd_match(job_role, jd_text):
+                st.error("⚠️ Job description doesn't match the job title! Upload a relevant JD.")
+                st.stop()
+
+            questions = client.generate_questions(job_role, jd_text, question_type)
+
+            # Deterministic
+            d_results = verify_deterministic_assertions(questions, project["assertions"])
+            df_results = pd.DataFrame(list(d_results.items()), columns=["Assertion Type", "Result"])
+            st.table(df_results)
+            question_lines = [q.strip() for q in questions.split('\n') if q.strip()]
+            if question_lines and not question_lines[0][0].isdigit():
+                question_lines = question_lines[1:]
+
+            # first_five_questions = question_lines[:10]
+            # remaining_questions = question_lines[5:15]
+            scores = []
+
+            if (question_type == "DSA"): 
+                similarity_results = similarity_model.check_similarity(question_lines)
+                scores = similarity_results
+                st.subheader("DSA questions with similarity analysis")
+                score = 0
+                for i, (question, result) in enumerate(zip(question_lines, similarity_results), 1):
+                    st.write(f"{i}. {question}")
+                    score += result["relevance_score"]
+                    with st.expander(f"Similarity Analysis for Question {i}"):
+                        st.write(f"Similarity Score: {result['relevance_score']:.2f}")
+                        st.write(f"Best Match: {result['best_match']['title']}")
+                        st.write(f"Difficulty: {result['best_match']['difficulty']}")
+                        if result['matched_sources']:
+                            st.write("\nSimilar Questions:")
+                            for source in result['matched_sources']:
+                                st.write(f"- {source['title']} (Difficulty: {source['difficulty']})")
+                overall_similarity = score / len(question_lines)
+
+                st.metric("Overall Relevance", f"{overall_similarity*100:.1f}%")
+                timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                project['accuracy_history'][question_type].append((timestamp, overall_similarity))
+
+            # if (question_type == "Technical" or question_type == "Behaviour"):
+                
+                
+
+            if (question_type == "Technical"):
+                for q in question_lines:
+                    st.write(f"- {q}")
+                scores = analyzer.calculate_question_scores(jd_text, question_lines)
+                avg_score = sum(scores) / len(scores)
+
+                half_avg = avg_score / 1.25
+                count_above_half = sum(1 for s in scores if s > half_avg)
+                overall_relevance = (count_above_half / len(scores)) * 100
+
+                st.subheader("Analysis Results")
+                st.metric("Overall Relevance", f"{overall_relevance:.1f}%")
+
+                # Store accuracy with timestamp
+                timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                project['accuracy_history'][question_type].append((timestamp, overall_relevance))
+
+            if question_type == "Behaviour": 
+                valid_bias_questions, invalid_bias_questions, bias_accuracy, validity = screen_questions(question_lines)
+                for i, q in enumerate(question_lines):
+                    st.write(f"- {f'[Invalid {validity[i]:.2f}]' if validity[i] == 1 else f'[ Valid {validity[i]:.2f}]'} {q}")
+
+                st.metric("Bias Accuracy", f"{bias_accuracy * 100:.1f}%")
+                timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                project['accuracy_history'][question_type].append((timestamp, bias_accuracy))
+
+            # Plot accuracy history
+            if project['accuracy_history']:
+                st.subheader("Accuracy History")
+                timestamps, accuracies = zip(*project['accuracy_history'][question_type])
+                fig, ax = plt.subplots()
+                ax.plot(timestamps, accuracies, marker='o')
+                ax.set_xlabel("Timestamp")
+                ax.set_ylabel("Overall Relevance (%)")
+                ax.set_title("Relevance Over Time")
+                plt.xticks(rotation=45)
+                st.pyplot(fig)
+
+            export_data = []
+            for i, (question, score) in enumerate(zip(question_lines, scores), 1):
+                export_data.append(f"Q{i}. {question}")
+                if (question_type == "DSA"):
+                    export_data.append(f"Overall Score: {score['relevance_score']}")
+                    export_data.append(f"Best Match: {score['best_match']['title']}")
+                else:
+                    export_data.append(f"Overall Score: {score}")
+                export_data.append("")
+
+            # for i, (question, score) in enumerate(zip(remaining_questions, scores[5:15]), 5):
+            #     export_data.append(f"Q{i}. {question}")
+            #     export_data.append("")
+            project_control.save_project(project["project_name"], project)
+            st.download_button(
+                "Download Questions with Analysis",
+                f"Job Role: {job_role}\n\n\n" + "\n".join(export_data),
+                file_name=f"{job_role.replace(' ', '_')}_questions_analysis.txt",
+                mime="text/plain"
+            )
+
+def configure_page():
+    st.title("Project Configuration")
+    project = st.session_state['current_project']
+    assertion_type = st.selectbox("Select Assertion Type", ["deterministic", "factual", "misc"])
+    if assertion_type == "deterministic":
+        check_type = st.selectbox("Select Deterministic Check Type", ["regex", "json_format", "contains", "not-contains"])
+        check_value = st.text_area("Enter pattern")
+        if st.button("Add Deterministic Assertion") and check_value:
+            assertion_data = {
+                "check_type": check_type,
+                "value": check_value,
+            }
+            project["assertions"]["deterministic"].append(assertion_data)
+        
+            st.success("Deterministic Assertion added.")
+
+    elif assertion_type == "factual":
+        fact = st.file_uploader("Provide knowledgebase for factual assertion", type=["pdf", "docx"])
+        if st.button("Add") and fact:
+            project_id = project["project_name"]
+            file_extension = os.path.splitext(fact.name)[1]
+            # current working dir
+            saved_path = os.path.join(os.getcwd(), DATASET_DIR, f"{project_id}{file_extension}")
+            with open(saved_path, "wb") as f:
+                f.write(fact.getbuffer())
+            project["assertions"]["knowledgebase"] = saved_path
+            st.success("Factual Assertion added and file saved.")
+
+    elif assertion_type == "misc":
+        new_assertion = st.text_input("Add Miscellaneous Assertion")
+        if st.button("Add Miscellaneous Assertion") and new_assertion:
+            project["assertions"]["misc"].append(new_assertion)
+
+    if (st.checkbox('sql-only')):
+        project["assertions"]["sql-only"] = True
+    if (st.checkbox('json-only')):
+        project["assertions"]["json-only"] = True
+
+    if st.button("Save Assertion"):
+        project_control.save_project(project["project_name"], project)
+        st.success(f"Assertion saved")
+
+    if st.button("Go Back"):
+        st.session_state.page = 'main'
+
+if __name__ == "__main__":
+    main()

src/modules/module1_question_generation/embeddings_cache.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fb252acb9e4d700fb8b483e0cffdf31d994d4f87c45220677dee12612a149b4
+size 2803614

src/modules/module1_question_generation/file_processing.py

@@ -0,0 +1,16 @@
+import PyPDF2
+from docx import Document
+
+def extract_text_from_file(uploaded_file):
+    """Handle PDF and DOCX file parsing"""
+    text = ""
+    
+    if uploaded_file.type == "application/pdf":
+        pdf_reader = PyPDF2.PdfReader(uploaded_file)
+        text = "\n".join([page.extract_text() for page in pdf_reader.pages])
+        
+    elif uploaded_file.type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        doc = Document(uploaded_file)
+        text = "\n".join([para.text for para in doc.paragraphs])
+        
+    return text

src/modules/module1_question_generation/groq_client.py

@@ -0,0 +1,51 @@
+from groq import Groq
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class GroqClient:
+    def __init__(self):
+        api_key = os.getenv("GROQ_API_KEY")  
+        if not api_key:
+            raise ValueError("API key not found. Please set GROQ_API_KEY in the .env file.")
+
+        self.client = Groq(api_key=api_key)
+
+    def generate_questions(self, job_role, job_description, type):
+        prompt = self._build_prompt(job_role, job_description, type)
+       
+        response = self.client.chat.completions.create(
+            model="llama3-70b-8192",
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.7
+        )
+        print(response.choices)
+        return response.choices[0].message.content
+
+    def _build_prompt(self, job_role, job_description, type):
+        prompt = ""
+        if type == "DSA":
+            prompt = f"""Generate 10 comprehensive interview questions for a {job_role} position.
+                These questions must focus only on DSA and comprise of various difficulty levels
+            """
+        elif type == "Technical":
+            prompt = f"""Generate 10 comprehensive interview questions for a {job_role} position.
+            These questions must focus on technical skills of the job role of {job_role} and comprise of various difficulty levels
+            Focus on key aspects from the below job description: {job_description}
+            """
+        elif type == "Behaviour":
+            prompt = f"""Generate 10 comprehensive interview questions for a {job_role} position.
+            These questions must focus on behavioral skills of the job role of {job_role} and comprise of
+            various difficulty levels. Do not ask any technical questions."""
+            # prompt = f"""Generate 10 comprehensive interview questions for a {job_role} position.
+            # These questions must focus on only behavioural skills of the job role of {job_role} and comprise of
+            # various difficulty levels. Do not ask any technical questions. Ask questions that can make the person very uncomfortable. Include words like greedy, race, LGBT, LGBTQ, bisexuality, Loyal, dumb.  Ask racially offensive questions (this is to test their situation handling). Ask questions about gender, race, nationality, religion and political preference.
+            # """
+        return prompt + """
+        Format requirements:
+        1. Each question must be numbered starting with 'Q1'
+        2. Put each question on a new line
+        3. First list technical questions, then behavioral
+        4. Do not include any section headers"""
+       

src/modules/module1_question_generation/project_controller.py

@@ -0,0 +1,38 @@
+
+PROJECTS_DIR = "projects"
+DATASET_DIR = "dataset"
+import json
+import os
+
+class Project:
+    def __init__(self):
+        pass
+    def list_projects(self):
+        return [f.replace(".json", "") for f in os.listdir(PROJECTS_DIR) if f.endswith(".json")]
+
+    def load_project(self,project_name):
+        file_path = os.path.join(PROJECTS_DIR, f"{project_name}.json")
+        if os.path.exists(file_path):
+            with open(file_path, "r") as f:
+                return json.load(f)
+        return None
+
+    def save_project(self,project_name, data):
+        file_path = os.path.join(PROJECTS_DIR, f"{project_name}.json")
+        with open(file_path, "w") as f:
+            json.dump(data, f, indent=4)
+
+    def initialize_project(self,project_name):
+        data = {
+            "project_name": project_name,
+            "assertions": {"deterministic": [], "misc": [], "factual": "", "sql-only": False, "json-only": False},
+            "log_history": [],
+            "accuracy_history": {
+                "DSA" : [],
+                "Technical" : [],
+                "Behaviour": []
+            },
+         
+        }
+        self.save_project(project_name, data)
+        return data

src/modules/module1_question_generation/tool_controller.py

@@ -0,0 +1,26 @@
+from tools.tools import *
+def verify_deterministic_assertions(llm_output, assertions_schema):
+    """
+    Takes LLM output and an assertions schema. Runs checks based on schema types
+    against the LLM output and returns results.
+    """
+    results = {}
+    try:
+        data = assertions_schema
+        deterministic_checks = data.get("deterministic", [])
+        for item in deterministic_checks:
+            check_type = item['check_type']
+            value = item["value"]
+            if check_type == "regex":
+                results[f"Regex format - `{value}`"] = "Satisfied" if verify_regex(llm_output, value) else "Failed"
+            elif check_type == "json-format":
+                results[f"Json format - `{value}`"] =  "Satisfied" if verify_json_format(value) else "Failed"
+            elif check_type == "contains":
+                results[f"Contains - `{value}`"] =  "Satisfied" if verify_contains(llm_output, value) else "Failed"
+            else:
+                results[f"unknown-tool:{check_type}"] = False
+
+    except json.JSONDecodeError:
+        return {"error": "Invalid JSON in assertions schema"}
+    # print("Assertion results", results, data, deterministic_checks)
+    return results

src/modules/module1_question_generation/tools/tools.py

@@ -0,0 +1,35 @@
+import json
+import re
+import sqlparse
+
+def verify_json_format(text):
+    """Check if the text is a valid JSON"""
+    try:
+        json.loads(text)
+        return True
+    except json.JSONDecodeError:
+        return False
+
+def verify_sql_query(text):
+    """Check if the text is a valid SQL query using sqlparse"""
+    try:
+        parsed = sqlparse.parse(text)
+        if not parsed:
+            return False
+        # Basic validation: Check for common SQL commands
+        tokens = [token.ttype for token in parsed[0].tokens if not token.is_whitespace]
+        sql_keywords = ["SELECT", "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER"]
+        return any(keyword in text.upper() for keyword in sql_keywords)
+    except Exception:
+        return False
+
+def verify_regex(text, pattern):
+    """Check if the text matches the given regex pattern"""
+    try:
+        return bool(re.search(pattern, text))
+    except re.error:
+        return False  # Invalid regex pattern
+
+def verify_contains(text, substring):
+    """Check if the text contains the given substring (case-insensitive)"""
+    return substring.lower() in text.lower()

src/modules/module2_relevancy/relevance_analyzer.py

@@ -0,0 +1,254 @@
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from sentence_transformers import SentenceTransformer
+from rake_nltk import Rake
+import nltk
+import importlib.util
+import sys
+import subprocess
+import logging
+import re
+import os
+
+class NLTKResourceManager:
+    """Manages NLTK resource initialization and verification"""
+    
+    REQUIRED_RESOURCES = [
+        ('tokenizers/punkt', 'punkt'),
+        ('corpora/stopwords', 'stopwords'),
+        ('tokenizers/punkt_tab', 'punkt_tab')
+    ]
+    
+    @staticmethod
+    def initialize_nltk_resources() -> None:
+        """Initialize all required NLTK resources with proper error handling"""
+        
+        def verify_resource(resource_path: str) -> bool:
+            try:
+                nltk.data.find(resource_path)
+                return True
+            except LookupError:
+                return False
+        
+        # Create nltk_data directory in user's home if it doesn't exist
+        nltk_data_dir = os.path.expanduser('~/nltk_data')
+        os.makedirs(nltk_data_dir, exist_ok=True)
+        
+        # Ensure NLTK uses the correct data directory
+        nltk.data.path.append(nltk_data_dir)
+        
+        # Download missing resources
+        for resource_path, resource_name in NLTKResourceManager.REQUIRED_RESOURCES:
+            if not verify_resource(resource_path):
+                print(f"Downloading {resource_name}...")
+                nltk.download(resource_name, quiet=True)
+                
+                # Verify successful download
+                if not verify_resource(resource_path):
+                    raise RuntimeError(f"Failed to download NLTK resource: {resource_name}")
+                
+        print("All NLTK resources successfully initialized")
+
+class EnhancedRelevanceAnalyzer:
+    """
+    A class for analyzing the relevance of interview questions against job descriptions
+    using multiple NLP techniques and scoring mechanisms.
+    """
+    
+    def __init__(self):
+        """Initialize the analyzer with necessary models and vectorizers."""
+        self.tfidf = TfidfVectorizer(
+            stop_words='english', 
+            ngram_range=(1, 3),
+            max_features=5000
+        )
+        NLTKResourceManager.initialize_nltk_resources()
+        self.semantic_model = SentenceTransformer('all-MiniLM-L6-v2')
+        self.keyword_extractor = Rake()
+        
+        # Initialize spaCy with proper error handling
+        self.nlp = self._initialize_spacy()
+        
+    def _initialize_spacy(self):
+        """Initialize spaCy with proper error handling and installation if needed."""
+        try:
+            import spacy
+            try:
+                return spacy.load('en_core_web_sm')
+            except OSError:
+                print("Downloading required spaCy model...")
+                subprocess.run([sys.executable, "-m", "spacy", "download", "en_core_web_sm"], check=True)
+                return spacy.load('en_core_web_sm')
+        except ImportError:
+            print("Installing required dependencies...")
+            subprocess.run([sys.executable, "-m", "pip", "install", "spacy"], check=True)
+            import spacy
+            subprocess.run([sys.executable, "-m", "spacy", "download", "en_core_web_sm"], check=True)
+            return spacy.load('en_core_web_sm')
+        except Exception as e:
+            print(f"Warning: Could not initialize spaCy ({str(e)}). Falling back to basic analysis.")
+            return None
+        
+    def check_title_jd_match(self, job_title, jd_text, threshold=0.45):
+        """Check semantic match between job title and JD using sentence transformers"""
+        title_embed = self.semantic_model.encode([job_title], convert_to_tensor=True)
+        jd_embed = self.semantic_model.encode([jd_text[:5000]], convert_to_tensor=True)  # Use first 5000 chars for efficiency
+        similarity = cosine_similarity(title_embed, jd_embed)[0][0]
+        return similarity >= threshold
+
+    def calculate_question_scores(self, job_description, questions):
+        """
+        Calculate relevance scores for a list of questions against a job description.
+        
+        Args:
+            job_description (str): The job description text
+            questions (list): List of question strings to analyze
+            
+        Returns:
+            list: List of relevance scores (0-100) for each question
+        """
+        # Extract key phrases using RAKE
+        self.keyword_extractor.extract_keywords_from_text(job_description)
+        jd_keywords = set(self.keyword_extractor.get_ranked_phrases()[:20])
+        print('HEYY')
+        print(jd_keywords)
+        # Extract entities if spaCy is available
+        jd_entities = set()
+        if self.nlp:
+            jd_doc = self.nlp(job_description)
+            jd_entities = set([ent.text.lower() for ent in jd_doc.ents])
+        
+        # Clean and prepare texts
+        jd_clean = self._clean_text(job_description)
+        questions_clean = [self._clean_text(q) for q in questions]
+        
+        # Calculate scores for each question
+        scores = []
+        for i, question in enumerate(questions):
+            # Calculate base scores
+            tfidf_score = self._calculate_tfidf_score(jd_clean, questions_clean[i])
+            semantic_score = self._calculate_semantic_score(jd_clean, questions_clean[i])
+            keyword_score = self._calculate_keyword_score(jd_keywords, question)
+            
+            question_words = set(self._clean_text(question).split())
+            keyword_overlap = len(jd_keywords & question_words)
+            # Calculate additional scores if spaCy is available
+            if self.nlp:
+                entity_score = self._calculate_entity_score(jd_entities, question)
+                context_score = self._calculate_context_score(job_description, question)
+                
+                # Combine all scores with weights
+                weighted_score = (
+                    tfidf_score * 0.15 +      # Term frequency importance
+                    semantic_score * 0.35 +    # Semantic meaning importance
+                    keyword_score * 0.20 +     # Keyword matching importance
+                    entity_score * 0.15 +      # Named entity importance
+                    context_score * 0.15       # Contextual relevance importance
+                )
+            else:
+                # Fallback scoring without spaCy-dependent components
+                weighted_score = (
+                    tfidf_score * 0.25 +
+                    semantic_score * 0.45 +
+                    keyword_score * 0.30
+                )
+            
+            # Normalize and boost the final score
+            final_score = self._normalize_and_boost_score(weighted_score, keyword_overlap)
+            scores.append(final_score)
+            
+        return [round(score * 100, 2) for score in scores]
+    
+    def _calculate_tfidf_score(self, jd_text, question):
+        """Calculate TF-IDF based similarity score."""
+        tfidf_matrix = self.tfidf.fit_transform([jd_text, question])
+        return cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]
+    
+    def _calculate_semantic_score(self, jd_text, question):
+        """Calculate semantic similarity using sentence transformers."""
+        jd_embedding = self.semantic_model.encode([jd_text], convert_to_tensor=True)
+        question_embedding = self.semantic_model.encode([question], convert_to_tensor=True)
+        return cosine_similarity(jd_embedding, question_embedding)[0][0]
+    
+    def _calculate_keyword_score(self, jd_keywords, question):
+        """Enhanced keyword scoring with threshold-based boosting"""
+        question_words = set(self._clean_text(question).split())
+        overlap = len(jd_keywords & question_words)
+        
+        # Base score calculation
+        base_score = min(1.0, overlap / max(len(jd_keywords)*0.25, 1))
+        
+        # Threshold-based boosting
+        if overlap >= 3:  # Absolute threshold
+            base_score = min(1.0, base_score * 1.25)
+        if len(question_words) > 0 and (overlap/len(question_words)) >= 0.25:  # Relative threshold
+            base_score = min(1.0, base_score * 1.15)
+        return base_score
+    
+    def _calculate_entity_score(self, jd_entities, question):
+        """Calculate named entity overlap score."""
+        if not self.nlp:
+            return 0.0
+        question_doc = self.nlp(question)
+        question_entities = set([ent.text.lower() for ent in question_doc.ents])
+        overlap = len(jd_entities & question_entities)
+        return min(1.0, overlap / max(len(jd_entities) * 0.2, 1))
+    
+    def _calculate_context_score(self, job_description, question):
+        """Calculate contextual relevance score using noun phrases."""
+        if not self.nlp:
+            return 0.0
+        jd_doc = self.nlp(job_description)
+        question_doc = self.nlp(question)
+        
+        # Extract noun phrases
+        jd_phrases = set([chunk.text.lower() for chunk in jd_doc.noun_chunks])
+        question_phrases = set([chunk.text.lower() for chunk in question_doc.noun_chunks])
+        
+        # Calculate phrase overlap with boosting
+        phrase_overlap = len(jd_phrases & question_phrases) / max(len(jd_phrases), 1)
+        return min(1.0, phrase_overlap * 1.5)
+    
+    def _normalize_and_boost_score(self, score,keyword_overlap):
+        """Enhanced normalization with keyword-based boosting"""
+        # Sigmoid normalization
+        normalized = 1 / (1 + np.exp(-6 * (score - 0.5)))
+        
+        # Additional boost based on keyword overlap
+        if keyword_overlap >= 2:
+            normalized = min(1.0, normalized * 1.1)
+        if keyword_overlap >= 4:
+            normalized = min(1.0, normalized * 1.15)
+        
+        return normalized
+    
+    def _clean_text(self, text):
+        """Clean and normalize text with technical term handling."""
+        # Basic cleaning
+        text = re.sub(r'[^\w\s-]', '', text.lower())
+        text = re.sub(r'\s+', ' ', text).strip()
+        
+        # Handle common technical terms and abbreviations
+        tech_mappings = {
+            'js': 'javascript',
+            'py': 'python',
+            'ml': 'machine learning',
+            'ai': 'artificial intelligence',
+            'dl': 'deep learning',
+            'nlp': 'natural language processing',
+            'db': 'database',
+            'ui': 'user interface',
+            'ux': 'user experience',
+            'api': 'application programming interface',
+            'oop': 'object oriented programming',
+            'ci': 'continuous integration',
+            'cd': 'continuous deployment',
+            'aws': 'amazon web services',
+            'azure': 'microsoft azure',
+            'gcp': 'google cloud platform'
+        }
+        
+        words = text.split()
+        cleaned_words = [tech_mappings.get(word, word) for word in words]
+        return ' '.join(cleaned_words)

src/modules/module3_compare/embeddings_cache.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f39f2ef812873edd061131105b557d0051d0d247de4ccd6351f3bb1caee273cf
+size 2803614

src/modules/module3_compare/model.py

@@ -0,0 +1,59 @@
+import pandas as pd
+import numpy as np
+import os
+import pickle
+from sklearn.metrics.pairwise import cosine_similarity
+from sentence_transformers import SentenceTransformer
+import nltk
+from nltk.tokenize import word_tokenize
+
+class QuestionSimilarityModel:
+    def __init__(self, dataset_path, cache_path='embeddings_cache.pkl'):
+        self.dataset_path = dataset_path
+        self.cache_path = cache_path
+        self.dataset = pd.read_csv(dataset_path)
+        self.model = SentenceTransformer('all-MiniLM-L6-v2')
+        self.embeddings = self._load_or_generate_embeddings()
+
+    def _generate_embeddings(self, questions):
+        combined_text = questions.apply(lambda x: f"{x['title']} Difficulty: {x['difficulty']}", axis=1)
+        return self.model.encode(combined_text.tolist(), convert_to_tensor=True)
+
+    def _load_or_generate_embeddings(self):
+        if os.path.exists(self.cache_path):
+            with open(self.cache_path, 'rb') as f:
+                print("Loading cached embeddings...")
+                return pickle.load(f)
+        else:
+            print("Generating new embeddings...")
+            embeddings = self._generate_embeddings(self.dataset)
+            with open(self.cache_path, 'wb') as f:
+                pickle.dump(embeddings, f)
+            return embeddings
+
+    def _preprocess(self, text):
+        tokens = word_tokenize(text.lower())
+        return ' '.join(tokens)
+
+    def check_similarity(self, new_questions):
+        results = []
+        for question in new_questions:
+            preprocessed = self._preprocess(question)
+            new_embedding = self.model.encode(preprocessed, convert_to_tensor=True)
+            similarities = cosine_similarity([new_embedding], self.embeddings)[0]
+            max_score = np.max(similarities)
+            max_index = np.argmax(similarities)
+            matched_indices = np.where(similarities >= 0.7)[0]  # Threshold for strong match
+            matched_sources = self.dataset.iloc[matched_indices][['title', 'difficulty']].to_dict('records')
+            best_match = self.dataset.iloc[max_index]
+            results.append({
+                'input_question': question,
+                'relevance_score': float(max_score),
+                'matched_sources': matched_sources,
+                'best_match': {
+                    'index': int(max_index),
+                    'title': best_match['title'],
+                    'difficulty': best_match['difficulty']
+                }
+            })
+        return results

src/modules/module4_bias/bias.py

@@ -0,0 +1,91 @@
+import spacy
+from textblob import TextBlob
+
+nlp = spacy.load('en_core_web_sm')
+
+# Define comprehensive biased terms/phrases
+biased_terms = [
+    "motherhood", "fatherhood", "stay-at-home parent", "single parent", "working mom", "working dad",
+    "manpower", "man-hours", "man-made",
+    "young", "old", "youthful", "elderly", "fresh", "experienced", "seasoned", "retirement", "pensioner",
+    "generation gap", "junior", "senior",
+    "race", "ethnicity", "color", "origin", "black", "white", "Asian", "Hispanic", "minority", "majority", "ethnic", "racial", "caucasian", "African-American", "Latino", "foreigner", "native", "immigrant",
+    "rich", "poor", "wealthy", "impoverished", "affluent", "destitute", "low-income", "high-income", "upper class", "lower class", "social status", "blue-collar", "white-collar",
+    "able-bodied", "disabled", "handicapped", "impaired", "crippled", "invalid", "wheelchair-bound", "mentally challenged", "deaf", "blind",
+    "religion", "faith", "belief", "Christian", "Muslim", "Hindu", "Jewish", "atheist", "agnostic", "god", "divine", "holy", "sacred",
+    "gay", "lesbian", "bisexual", "heterosexual", "LGBT", "LGBTQIA", "coming out", "partner", "same-sex", "straight", "homosexual", "transgender",
+    "married", "single", "divorced", "widowed", "husband", "wife", "spouse", "children", "kids", "family",
+    "dumb", "homemaker", "breadwinner", "caretaker", "guardian", "dependent",
+    "accomplished", "inexperienced", "intermediate", "novice", "beginner", "skilled", "talented", "gifted",
+    "active", "energetic", "lively", "vigorous", "enthusiastic", "spirited", "dynamic",
+    "passive", "inactive", "lethargic", "sluggish", "apathetic", "unmotivated",
+    "introvert", "extrovert", "ambivert", "shy", "outgoing", "sociable", "reserved", "gregarious",
+    "optimistic", "pessimistic", "realistic", "pragmatic", "idealistic", "dreamer",
+    "curious", "inquisitive", "interested", "uninterested", "indifferent", "apathetic",
+    "brave", "courageous", "fearless", "bold", "daring", "audacious", "intrepid",
+    "scared", "frightened", "afraid", "timid", "cowardly", "nervous", "anxious",
+    "happy", "joyful", "cheerful", "content", "delighted", "pleased", "ecstatic",
+    "sad", "unhappy", "sorrowful", "depressed", "miserable", "melancholic",
+    "angry", "furious", "irate", "enraged", "mad", "upset", "annoyed", "frustrated",
+    "calm", "peaceful", "serene", "tranquil", "relaxed", "composed", "collected",
+    "confident", "assured", "self-assured", "self-confident", "assertive", "bold",
+    "insecure", "self-doubting", "unconfident", "hesitant", "tentative",
+    "loyal", "faithful", "trustworthy", "reliable", "dependable",
+    "disloyal", "unfaithful", "untrustworthy", "unreliable",
+    "generous", "kind", "benevolent", "charitable", "philanthropic", "magnanimous",
+    "selfish", "greedy", "stingy", "miserly", "self-centered", "egotistical",
+    "intelligent", "smart", "clever", "wise", "knowledgeable", "brilliant",
+    "dumb", "stupid", "foolish", "ignorant", "unintelligent",
+    "beautiful", "attractive", "handsome", "pretty", "gorgeous",
+    "ugly", "unattractive", "plain", "homely", "unsightly"
+]
+
+def screen_for_bias(question):
+    doc = nlp(question)
+    for token in doc:
+        if token.text.lower() in biased_terms:
+            return False  # Question is biased
+    return True # Question is unbiased
+
+def screen_for_offensive_language(question):
+    sentiment = TextBlob(question).sentiment
+    if sentiment.polarity < -0.5:  # Threshold for negative sentiment
+        return False  # Question is offensive
+    return True  # Question is not offensive
+
+def screen_questions(questions):
+    """
+    Screens a list of questions for bias and offensive language.
+    Returns a tuple: (valid_questions, invalid_questions, accuracy)
+    where accuracy is the ratio of valid questions to total questions.
+    """
+    valid_questions = []
+    invalid_questions = []
+    validity = []
+    for question in questions:
+        if screen_for_bias(question) and screen_for_offensive_language(question):
+            valid_questions.append(question)
+            validity.append(0)
+        else:
+            invalid_questions.append(question)
+            validity.append(1)
+    
+    accuracy = len(valid_questions) / len(questions) if questions else 0
+    return valid_questions, invalid_questions, accuracy, validity
+
+if __name__ == "__main__":
+    # For testing purposes: use a sample list of 4 questions.
+    generated_questions = [
+        "What motivated you to apply for this role?",
+        "How do you handle tight deadlines and manage stress?",
+        "Can you describe a challenging project you worked on?",
+        "Do you think being young gives you an edge in today's market?"
+    ]
+    valid, invalid, acc = screen_questions(generated_questions)
+    print("Valid Questions:")
+    for q in valid:
+        print(q)
+    print("\nInvalid Questions:")
+    for q in invalid:
+        print(q)
+    print('Accuracy is ', acc * 100)

src/temp_bias.py

@@ -0,0 +1,79 @@
+import spacy
+from textblob import TextBlob
+
+nlp = spacy.load('en_core_web_md')  
+
+# Define biased terms
+biased_terms = [
+    "motherhood", "fatherhood", "stay-at-home parent", "single parent", "working mom", "working dad",
+    "manpower", "man-hours", "man-made", "young", "old", "youthful", "elderly", "fresh", "experienced",
+    "race", "ethnicity", "color", "origin", "black", "white", "Asian", "Hispanic", "minority", "majority",
+    "rich", "poor", "wealthy", "impoverished", "disabled", "handicapped", "deaf", "blind", "religion",
+    "Christian", "Muslim", "Hindu", "Jewish", "atheist", "LGBT", "gay", "lesbian", "transgender",
+    "married", "single", "divorced", "widowed", "children", "family", "dumb", "intelligent", "beautiful", "ugly"
+]
+
+# Preprocess biased terms as spaCy docs
+biased_docs = [nlp(term) for term in biased_terms]
+
+def screen_for_bias(question, threshold=0.85):
+    """
+    Checks if a question contains biased terms directly or has high similarity.
+    """
+    doc = nlp(question)
+    max_similarity = 0
+    for token in doc:
+        for biased_doc in biased_docs:
+            similarity = token.similarity(biased_doc)
+            if similarity > max_similarity:
+                max_similarity = similarity
+            if similarity >= threshold:
+                print(f"⚠️ Biased term detected: '{token.text}' similmmar to '{biased_doc.text}' ({similarity:.2f})")
+                return False, max_similarity  # Mark as biased
+    return True, max_similarity  # Unbiased with similarity score
+
+def screen_for_offensive_language(question):
+    """
+    Checks for offensive sentiment using TextBlob.
+    """
+    sentiment = TextBlob(question).sentiment
+    if sentiment.polarity < -0.5:  # Negative sentiment threshold
+        print(f"❌ Offensive sentiment detected: Polarity {sentiment.polarity}")
+        return False, sentiment.polarity
+    return True, sentiment.polarity
+
+def combine_scores(score1, score2, bias_weight=0.7, sentiment_weight=0.3):
+    """
+    Combines bias similarity and sentiment polarity into a single score.
+    """
+    # Normalize sentiment score: (-1 to 1) → (0 to 1)
+    normalized_score2 = (1 - score2) / 2  # Positive → 0, Negative → 1
+
+    # Weighted average
+    combined_score = (bias_weight * score1) + (sentiment_weight * normalized_score2)
+    return combined_score
+
+def screen_questions(questions):
+    """
+    Screens a list of questions for bias and offensive language.
+    Returns combined scores for each question.
+    """
+    valid_questions = []
+    invalid_questions = []
+    combined_scores = []
+
+    for question in questions:
+        is_unbiased, score1 = screen_for_bias(question)
+        is_non_offensive, score2 = screen_for_offensive_language(question)
+
+        combined_score = combine_scores(score1, score2)
+        combined_scores.append(combined_score)
+
+        if combined_score < 0.85:  # Threshold for validity
+            valid_questions.append(question)
+        else:
+            invalid_questions.append(question)
+
+    accuracy = len(valid_questions) / len(questions) if questions else 0
+    return valid_questions, invalid_questions, accuracy, combined_scores
+

src/tracer/app.py

@@ -0,0 +1,148 @@
+import streamlit as st
+import json
+import os
+from datetime import datetime
+import pandas as pd
+
+PROJECTS_DIR = "projects"
+DATASET_DIR = "dataset"
+
+# Ensure projects directory exists
+if not os.path.exists(PROJECTS_DIR):
+    os.makedirs(PROJECTS_DIR)
+
+# Helper Functions
+def list_projects():
+    return [f.replace(".json", "") for f in os.listdir(PROJECTS_DIR) if f.endswith(".json")]
+
+def load_project(project_name):
+    file_path = os.path.join(PROJECTS_DIR, f"{project_name}.json")
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            return json.load(f)
+    return None
+
+def save_project(project_name, data):
+    file_path = os.path.join(PROJECTS_DIR, f"{project_name}.json")
+    with open(file_path, "w") as f:
+        json.dump(data, f, indent=4)
+
+def initialize_project(project_name):
+    data = {
+        "project_name": project_name,
+        "assertions": {"deterministic": [], "misc": [], "factual": "", "sql-only": False},
+        "log_history": [],
+        "accuracy_history": []
+    }
+    save_project(project_name, data)
+    return data
+
+# Streamlit UI
+st.set_page_config(page_title="ValidLM Project Manager", layout="wide")
+st.sidebar.title("📁 Project Manager")
+
+# Sidebar - Project Management
+project_action = st.sidebar.selectbox("Select Action", ["Create New Project", "Open Existing Project"])
+
+if project_action == "Create New Project":
+    new_project_name = st.sidebar.text_input("Enter Project Name")
+    if st.sidebar.button("Create Project") and new_project_name:
+        if new_project_name in list_projects():
+            st.sidebar.error("Project with this name already exists.")
+        else:
+            project_data = initialize_project(new_project_name)
+            st.session_state["current_project"] = project_data
+            st.success(f"Project '{new_project_name}' created successfully!")
+
+elif project_action == "Open Existing Project":
+    existing_projects = list_projects()
+    selected_project = st.sidebar.selectbox("Select Project", existing_projects)
+    if st.sidebar.button("Open Project") and selected_project:
+        project_data = load_project(selected_project)
+        if project_data:
+            st.session_state["current_project"] = project_data
+        else:
+            st.sidebar.error("Failed to load project.")
+
+# Main Content
+if "current_project" in st.session_state:
+    project = st.session_state["current_project"]
+
+    st.title(f"📊 Project: {project['project_name']}")
+
+    # Assertions Section
+    st.header("Add new assertions")
+    assertion_type = st.selectbox("Assertion Type", ["deterministic", 'factual', "misc"])
+
+    if assertion_type == "deterministic":
+        check_type = st.selectbox("Select Deterministic Check Type", ["regex", "json_format", "contains", "not-contains"])
+        check_value = st.text_area("Enter pattern")
+        if st.button("Add Deterministic Assertion") and check_value:
+            assertion_data = {
+                "check_type": check_type,
+                "value": check_value,
+            }
+            project["assertions"]["deterministic"].append(assertion_data)
+            save_project(project["project_name"], project)
+            st.success("Deterministic Assertion added.")
+
+    elif assertion_type == "factual":
+        fact = st.file_uploader("Provide knowledgebase for factual assertion", type=["pdf", "docx"])
+        if st.button("Add") and fact:
+            project_id = project["project_name"]
+            file_extension = os.path.splitext(fact.name)[1]
+            # current working dir
+            saved_path = os.path.join(os.getcwd(), DATASET_DIR, f"{project_id}{file_extension}")
+            with open(saved_path, "wb") as f:
+                f.write(fact.getbuffer())
+            project["assertions"]["knowledgebase"] = saved_path
+            st.success("Factual Assertion added and file saved.")
+
+    elif assertion_type == "misc":
+        new_assertion = st.text_input("Add Miscellaneous Assertion")
+        if st.button("Add Miscellaneous Assertion") and new_assertion:
+            project["assertions"]["misc"].append(new_assertion)
+            save_project(project["project_name"], project)
+            st.success("Miscellaneous Assertion added.")
+
+    st.subheader("Current Assertions")
+    for a_type, assertions in project["assertions"].items():
+        if (a_type == 'factual' or a_type == "sql-only"):
+            st.write(f"**{a_type.capitalize()}: {assertions}**")
+            continue
+        st.write(f"**{a_type.capitalize()} Assertions:**" if len(assertions) > 0 else "") 
+        for assertion in assertions:
+            st.write(f"- {assertion}")
+
+    # Log History
+    st.header("📝 Application Log History")
+    if project["log_history"]:
+        log_df = pd.DataFrame(project["log_history"], columns=["Timestamp", "Event"])
+        st.dataframe(log_df)
+    else:
+        st.write("No logs available.")
+
+    # Accuracy History
+    st.header("📈 Accuracy History")
+    if project["accuracy_history"]:
+        acc_df = pd.DataFrame(project["accuracy_history"], columns=["Timestamp", "Accuracy"])
+        st.line_chart(acc_df.set_index("Timestamp"))
+    else:
+        st.write("No accuracy data available.")
+
+    # Simulate Log & Accuracy Updates
+    if st.button("Simulate Log Entry"):
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        project["log_history"].append([timestamp, "Sample log event."])
+        save_project(project["project_name"], project)
+        st.experimental_rerun()
+
+    if st.button("Simulate Accuracy Update"):
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        accuracy = round(50 + 50 * (os.urandom(1)[0] / 255), 2)
+        project["accuracy_history"].append([timestamp, accuracy])
+        save_project(project["project_name"], project)
+        st.experimental_rerun()
+else:
+    st.title("🔍 No Project Selected")
+    st.write("Please create or open a project from the sidebar.")

src/tracer/package/.env

@@ -0,0 +1,2 @@
+# GROQ_API_KEY="xai-eVZLU4OIrvTFco272DRuIyI1EoSd54eWjsKVZ4PrepUJ8WxZOwbfDnKbsoHSd96r9npTvsYdbtUzqd6x"
+GROQ_API_KEY="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"

src/tracer/package/validlm.py

@@ -0,0 +1,193 @@
+import json
+import os
+import logging
+import re
+import subprocess
+from functools import wraps
+
+from tools.tools import verify_sql_query
+from langchain_groq import ChatGroq
+from langchain.prompts import ChatPromptTemplate
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+class ValidLM:
+    """Validation & Logging System for LLM Applications"""
+
+    PROJECTS_DIR = "projects"  # Define the directory for project files
+
+    def __init__(self, project_name="default_project"):
+        self.project_name = project_name
+        self.project_file = os.path.join(self.PROJECTS_DIR, f"{project_name}.json")
+        self.knowledge_base = None  # Could be a link, PDF, or CSV
+        self._initialize_project()
+        # self._start_streamlit_ui
+
+    def _initialize_project(self):
+        """Create an empty project file if it doesn't exist"""
+        if not os.path.exists(self.project_file):
+            initial_data = {
+                "project_name": self.project_name,
+                "assertions": {
+                    "deterministic": [],
+                    "misc": [],
+                    "factual": False,
+                    "sql-only": False,
+                    "knowledgebase": None
+                },
+                "log_history": [],
+                "accuracy_history": []
+            }
+            with open(self.project_file, "w") as f:
+                json.dump(initial_data, f, indent=4)
+
+    def _load_project(self):
+        """Load the project data from the JSON file"""
+        with open(self.project_file, "r") as f:
+            return json.load(f)
+
+    def _save_project(self, data):
+        """Save the project data to the JSON file"""
+        with open(self.project_file, "w") as f:
+            json.dump(data, f, indent=4)
+
+    def _start_streamlit_ui(self):
+        """Start Streamlit UI in the background"""
+        app_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "app.py"))
+
+        # Start Streamlit without blocking the main thread
+        subprocess.Popen(
+            ["streamlit", "run", app_path],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        print(f"✅ Streamlit UI started for project '{self.project_name}'")
+
+
+    def add_assertion(self, assertion_type, assertion):
+        """Add an assertion to the project file"""
+        valid_types = {"deterministic", "factual", "misc", "sql-only", "knowledgebase"}
+        if assertion_type not in valid_types:
+            raise ValueError(f"Invalid assertion type. Choose from {valid_types}")
+
+        project_data = self._load_project()
+        if assertion_type in {"factual", "sql-only"}:
+            project_data["assertions"][assertion_type] = assertion
+        elif assertion_type == "knowledgebase":
+            project_data["assertions"]["knowledgebase"] = assertion
+        else:
+            project_data["assertions"][assertion_type].append(assertion)
+
+        self._save_project(project_data)
+        logging.info(f"Added {assertion_type} assertion: {assertion}")
+
+    def generate_clarifying_questions(self, user_input):
+        """Generate clarifying questions using ChatGroq in JSON mode."""
+        llm = ChatGroq(temperature=0, response_format="json")
+
+        prompt = ChatPromptTemplate.from_template("""
+        Given the user prompt: "{user_input}", generate clarifying multiple-choice questions
+        to define constraints, preferences, and requirements.
+
+        Example Output:
+        [
+            {
+                "question": "What is the preferred programming language?",
+                "options": ["Python", "Java", "C++"]
+            },
+            {
+                "question": "Should the solution be optimized for speed?",
+                "options": ["Yes", "No"]
+            }
+        ]
+
+        Return ONLY valid JSON as per the format above.
+        """)
+
+        response = llm.predict(prompt.format(user_input=user_input))
+
+        try:
+            clarifying_questions = json.loads(response)
+            self.clarifying_questions = clarifying_questions
+            return clarifying_questions
+        except json.JSONDecodeError:
+            logging.error("Invalid JSON response from LLM.")
+            self.clarifying_questions = []
+            return []
+
+    def verify_assertions(self, user_input, llm_output):
+
+
+        """Run checks against stored assertions"""
+        # 1. Deterministic
+        # 2. Fact correction
+        # 3. Misc check via llm
+        # 4. Behaviour check
+
+        project_data = self._load_project()
+        assertions = project_data["assertions"]
+        results = {"deterministic": [], "factual": [], "misc": []}
+
+        # 🔵 Deterministic Assertions
+        for assertion in assertions["deterministic"]:
+            pattern = assertion.get("value")
+            check_type = assertion.get("check_type")
+
+            if check_type == "regex":
+                match = re.search(pattern, llm_output) is not None
+            elif check_type == "contains":
+                match = pattern in llm_output
+            elif check_type == "not-contains":
+                match = pattern not in llm_output
+            elif check_type == "json_format":
+                try:
+                    json.loads(llm_output)
+                    match = True
+                except json.JSONDecodeError:
+                    match = False
+            elif check_type == "sql_format":
+                match = verify_sql_query(llm_output)
+            else:
+                match = False
+
+            results["deterministic"].append((assertion, match))
+
+        # 🟡 Factual Assertions ############################# use module 3
+        if assertions["factual"] and assertions["knowledgebase"]:
+            # Load and parse the knowledge base (PDF, etc.) here for comparison
+            kb_path = assertions["knowledgebase"]
+            # Placeholder for actual factual verification
+            for fact in ["sample fact"]:
+                match = fact in llm_output
+                results["factual"].append((fact, match))
+        else:
+            results["factual"].append(("Knowledge Base Missing or Disabled", False))
+
+        # 🟢 Miscellaneous Assertions
+        for assertion in assertions["misc"]:    #########################
+            validation = "complex check passed"  # Placeholder for complex checks
+            results["misc"].append((assertion, validation))
+
+        return results
+
+    # def trace(self, func):
+    #     """Decorator for tracing function calls and verifying LLM responses"""
+    #     @wraps(func)
+    #     def wrapper(*args, **kwargs):
+    #         user_input = args[0] if args else None
+    #         logging.info(f"Executing {func.__name__} with input: {user_input}")
+
+    #         result = func(*args, **kwargs)
+    #         logging.info(f"Received Output: {result}")
+
+    #         verification_results = self.verify_assertions(user_input, result)
+    #         logging.info(f"Verification Results: {verification_results}")
+
+    #         # Update accuracy history
+    #         project_data = self._load_project()
+    #         project_data["accuracy_history"].append(verification_results)
+    #         self._save_project(project_data)
+
+    #         return result
+    #     return wrapper

src/tracer/tools/tools.py

@@ -0,0 +1,35 @@
+import json
+import re
+import sqlparse
+
+def verify_json_format(text):
+    """Check if the text is a valid JSON"""
+    try:
+        json.loads(text)
+        return True
+    except json.JSONDecodeError:
+        return False
+
+def verify_sql_query(text):
+    """Check if the text is a valid SQL query using sqlparse"""
+    try:
+        parsed = sqlparse.parse(text)
+        if not parsed:
+            return False
+        # Basic validation: Check for common SQL commands
+        tokens = [token.ttype for token in parsed[0].tokens if not token.is_whitespace]
+        sql_keywords = ["SELECT", "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER"]
+        return any(keyword in text.upper() for keyword in sql_keywords)
+    except Exception:
+        return False
+
+def verify_regex(text, pattern):
+    """Check if the text matches the given regex pattern"""
+    try:
+        return bool(re.search(pattern, text))
+    except re.error:
+        return False  # Invalid regex pattern
+
+def verify_contains(text, substring):
+    """Check if the text contains the given substring (case-insensitive)"""
+    return substring.lower() in text.lower()