Push new changes

evaluations/repo_evaluations.py

@@ -10,15 +10,11 @@ from huggingface_hub import InferenceClient
 API_URL = "https://api-inference.huggingface.co/models/openlm-research/open_llama_3b_v2"
 headers = {"Authorization": "Bearer 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", "x-wait-for-model": "true"}
 
-
 client = InferenceClient(
     "meta-llama/Llama-3.1-8B-Instruct",
     token="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",
 )
 
-def init_llm(verbose):
-  log(verbose, "LOG", "Initializing LLM...")
-
 def evaluate(llm, verbose, repo_url, title=None, year=None):
   repository_zip_name = "data/repo.zip"
   token = os.getenv("githubToken")
@@ -35,11 +31,6 @@ def evaluate(llm, verbose, repo_url, title=None, year=None):
       if (get_api_link(repo_url) != ""):
           results["pred_valid"] = True
       else:
-          results["pred_live"] = "No"
-          results["pred_training"] = "No"
-          results["pred_evaluation"] = "No"
-          results["pred_weights"] = "No"
-          results["pred_packages"] = "No"
           return results
 
       username, repo_name = decompose_url(repo_url)
@@ -86,17 +77,13 @@ def evaluate(llm, verbose, repo_url, title=None, year=None):
       results["pred_live"] = "No"
       return results
 
-def full_evaluations():
+def full_evaluation():
   paper_dump = pd.read_csv("data/dump.csv", sep="\t")
-  repro = evaluate(None, False)
   full_results = []
 
-  nth = 1
   for idx, row in paper_dump.iterrows():
-      if (idx % nth != 0):
-          continue
 
-      if (row["url"] == ""):
+      if (pd.isna(row["url"]) | (row["url"] == "")):
           continue
 
       print(str(int(100 * idx / paper_dump["title"].count())) + "% done")
@@ -105,6 +92,7 @@ def full_evaluations():
           row[column] = result[column]
 
       full_results.append(row)
+  return pd.dataFrame(full_results)
 
 def midl_evaluations():
   compare_to_gt = True
@@ -138,21 +126,23 @@ def midl_evaluations():
       full_results.append(row)
       if (compare_to_gt):
           print("\nSummary:")
-          if ((row["pred_dependencies"] is not None) & (row["dependencies"] != "")):
+          if ((~pd.isna(row["dependencies"])) & (row["pred_dependencies"] is not None)):
               eval_dependencies.append(row["pred_dependencies"] == row["dependencies"])
               print(f"Dependencies acc. - {row['pred_dependencies']} (GT:{row['dependencies']}) / {int(100 * np.mean(eval_dependencies))}%")
-          if ((row["pred_training"] is not None) & (row["training"] != "")):
+          if ((~pd.isna(row["training"])) & (row["pred_dependencies"] is not None)):
               eval_training.append(row["training"] == row["pred_training"])
               print(f"Training acc. -{row['pred_training']} (GT:{row['training']}) / {int(100 * np.mean(eval_training))}%")
-          if ((row["pred_evaluation"] is not None) & (row["evaluation"] != "")):
+          if ((~pd.isna(row["evaluation"])) & (row["pred_dependencies"] is not None)):
               eval_evaluating.append(row["evaluation"] == row["pred_evaluation"])
               print(f"Evaluating acc. - {row['pred_evaluation']} (GT:{row['evaluation']}) / {int(100 * np.mean(eval_evaluating))}%")
-          if ((row["pred_weights"] is not None) & (row["weights"] != "")):
+          if ((~pd.isna(row["weights"])) & (row["pred_dependencies"] is not None)):
               eval_weights.append(row["weights"] == row["pred_weights"])
               print(f"Weights acc. - {row['pred_weights']} (GT:{row['weights']}) / {int(100 * np.mean(eval_weights))}%")
-          if ((row["pred_readme"] is not None) & (row["readme"] != "")):
+          if ((~pd.isna(row["readme"])) & (row["pred_dependencies"] is not None)):
               eval_readme.append(row["readme"] == row["pred_readme"])
               print(f"README acc. - {row['pred_readme']} (GT:{row['readme']}) / {int(100 * np.mean(eval_readme))}%")
-          if ((row["pred_license"] is not None) & (row["license"] != "")):
+          if ((~pd.isna(row["license"])) & (row["pred_dependencies"] is not None)):
               eval_licensing.append(("No" if row["license"] == "No" else "Yes") == row["pred_license"])
-              print(f"LICENSE acc. - {row['pred_license']} (GT:{row['license']}) / {int(100 * np.mean(eval_licensing))}%")
+              print(f"LICENSE acc. - {row['pred_license']} (GT:{row['license']}) / {int(100 * np.mean(eval_licensing))}%")
+
+  return pd.dataFrame(full_results)

evaluations/utils.py

@@ -108,7 +108,10 @@ def log(verbose, log_type, log_text, hf=False):
 
     # Align line-break
     if (log_text.startswith("\n")):
-        print("\n")
+        if (verbose == 1):
+            print("\n")
+        if (verbose == 2):
+            st.write("\n")
         log_text = log_text.lstrip('\n')
 
     if (log_type == "LOG"):
@@ -127,4 +130,7 @@ def log(verbose, log_type, log_text, hf=False):
         st.write(log_text)
         return
 
-    raise Exception(log_text)
+    raise Exception(log_text)
+
+def init_llm(verbose):
+  log(verbose, "LOG", "Initializing LLM...")

evaluations/validating.py

@@ -8,14 +8,12 @@ def evaluate(verbose, llm, zip, readme):
       'tensorflow': [
           r'tf\.keras\.models\.load_model',  # TensorFlow model loading
           r'tf\.saved_model\.load',
-          r'model\.predict',  # Running inference
-          r'model\(.+\)'  # Direct model invocation for inference
+          r'\.predict',  # Running inference
       ],
       'pytorch': [
           r'torch\.load',  # PyTorch model loading
           r'torch\.jit\.load',  # PyTorch JIT model loading
-          r'model\.eval',  # Running inference
-          r'model\(.+\)'  # Direct model invocation for inference
+          r'\.eval',  # Running inference
       ]
   }
 

full_eval.py

@@ -0,0 +1,11 @@
+from evaluations.repo_evaluations import full_evaluation
+# importing os module for environment variables
+import os
+# importing necessary functions from dotenv library
+from dotenv import load_dotenv 
+# loading variables from .env file
+load_dotenv() 
+token = os.getenv("githubToken")
+
+res = full_evaluation()
+res.to_csv("results.csv", sep="\t", index=False)

midl.py

@@ -2,9 +2,10 @@ from evaluations.repo_evaluations import midl_evaluations
 # importing os module for environment variables
 import os
 # importing necessary functions from dotenv library
-from dotenv import load_dotenv, dotenv_values 
+from dotenv import load_dotenv 
 # loading variables from .env file
 load_dotenv() 
 token = os.getenv("githubToken")
 
-midl_evaluations()
+res = midl_evaluations()
+res.to_csv("results_midl.csv", sep="\t", index=False)