Upload app_federal.py

app_federal.py

@@ -26,6 +26,7 @@ from huggingface_hub import CommitScheduler
 from ast import literal_eval
 import re
 import requests
+#from huggingface_hub import hf_hub_download
 
 
 def run_courtlistener_api(casename, citation, court):
@@ -276,7 +277,7 @@ def run_retrieval(query):
     print ("query", query)
 
     query_embeddings = run_dense_retrieval(query)
-    query_embeddings = pca_model.transform(query_embeddings)
+    #query_embeddings = pca_model.transform(query_embeddings)
     D, I = faiss_index.search(query_embeddings, 100)
     scores_embeddings = D[0]
     indices_embeddings = I[0]
@@ -322,14 +323,26 @@ extractive_qa = pipeline("question-answering", model="ai-law-society-lab/extract
 
 ds_paragraphs = load_dataset("ai-law-society-lab/federal-caselaw-paragraphs", token=os.getenv('hf_token'))["train"]
 
+"""
 ds = load_dataset("ai-law-society-lab/federal-caselaw-embeddings-PCA-768", token=os.getenv('hf_token'))["train"]
 ds = ds.with_format("np")
 faiss_index = load_faiss_index(ds["embeddings"])
+"""
+
+#     repo_id = "ai-law-society-lab/save_OPD_project_output"
+#    url = "https://huggingface.co/datasets/ai-law-society-lab/save_OPD_project_output"
+
+url = "https://huggingface.co/datasets/ai-law-society-lab/autofaiss-federal-index/"
+faiss_index = "/Users/ds8100/Documents/NJ-caselaw-index/federal-index-faiss/knn.index"
+file_path = hf_hub_download(repo_id=repo_id, filename="knn.index", repo_type="dataset", token=hf_token)
 
+index = faiss.read_index(file_path)
 
+
+"""
 with open('PCA_model.pkl', 'rb') as f:
     pca_model = pickle.load(f)
-
+"""
 
 with open("Federal_caselaw_metadata.json") as f:
     metadata = json.load(f)