Upload 3 files

app.py

@@ -27,28 +27,59 @@ time_taken_for_response = 'N/A'
 st.subheader("Hi, What do you want to know today?")
 question = st.text_area("Enter your question:", placeholder="Type your question here...", height=100)
 
+# # Submit Button
+# if st.button("Submit"):
+#     start_time = time.time()
+#     retrieved_documents = retrieve_documents_hybrid(question, 10)  
+#     response = generate_response_from_document(question, retrieved_documents)
+#     end_time = time.time()
+#     time_taken_for_response = end_time-start_time
+# else:
+#     response = ""
+
+# # Response Section
+# st.subheader("Response")
+# st.text_area("Generated Response:", value=response, height=150, disabled=True)
+
+# # Metrics Section
+# st.subheader("Metrics")
+
+# col1, col2 = st.columns([1, 3])  # Creating two columns for button and metrics display
+
+# with col1:
+#     if st.button("Calculate Metrics"):
+#         metrics = calculate_metrics(question, response, retrieved_documents, time_taken_for_response)
+#     else:
+#         metrics = ""
+
+# with col2:
+#     st.text_area("Metrics:", value=metrics, height=100, disabled=True)
+
+if "retrieved_documents" not in st.session_state:
+    st.session_state.retrieved_documents = []
+if "response" not in st.session_state:
+    st.session_state.response = ""
+if "time_taken_for_response" not in st.session_state:
+    st.session_state.time_taken_for_response = "N/A"
+
 # Submit Button
 if st.button("Submit"):
     start_time = time.time()
-    retrieved_documents = retrieve_documents_hybrid(question, 10)  
-    response = generate_response_from_document(question, retrieved_documents)
+    st.session_state.retrieved_documents = retrieve_documents_hybrid(question, 10)  
+    st.session_state.response = generate_response_from_document(question, st.session_state.retrieved_documents)
     end_time = time.time()
-    time_taken_for_response = end_time-start_time
-else:
-    response = ""
+    st.session_state.time_taken_for_response = end_time - start_time
 
-# Response Section
+# Display stored response
 st.subheader("Response")
-st.text_area("Generated Response:", value=response, height=150, disabled=True)
-
-# Metrics Section
-st.subheader("Metrics")
+st.text_area("Generated Response:", value=st.session_state.response, height=150, disabled=True)
 
 col1, col2 = st.columns([1, 3])  # Creating two columns for button and metrics display
 
+# Calculate Metrics Button
 with col1:
     if st.button("Calculate Metrics"):
-        metrics = calculate_metrics(question, response, retrieved_documents, time_taken_for_response)
+        metrics = calculate_metrics(question, st.session_state.response, st.session_state.retrieved_documents, st.session_state.time_taken_for_response)
     else:
         metrics = ""
 

retrieval.py

@@ -4,6 +4,9 @@ from langchain.schema import Document
 import faiss
 from rank_bm25 import BM25Okapi
 from data_processing import embedding_model #, index, actual_docs
+from sentence_transformers import CrossEncoder
+
+reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 
 retrieved_docs = None
 
@@ -36,8 +39,10 @@ def retrieve_documents_hybrid(query, top_k=5):
 
     # Merge FAISS + BM25 Results
     retrieved_docs = list(set(faiss_docs + bm25_docs))[:top_k]
+    
+    reranked_docs = rerank_documents(query, retrieved_docs)
 
-    return retrieved_docs
+    return reranked_docs
 
 # Retrieval Function
 def retrieve_documents(query, top_k=5):
@@ -80,3 +85,8 @@ def find_query_dataset(query):
     best_dataset = dataset_names[nearest_index[0][0]]
     return best_dataset
 
+def rerank_documents(query, retrieved_docs):
+    doc_texts = [doc for doc in retrieved_docs]
+    scores = reranker.predict([[query, doc] for doc in doc_texts])
+    ranked_docs = [doc for _, doc in sorted(zip(scores, retrieved_docs), reverse=True)]
+    return ranked_docs[:5]  # Return top k most relevant