new files, posible model

app.py

@@ -1,7 +1,10 @@
 import streamlit as st
 from evaluations.repo_evaluations import evaluate
+from evaluations.models import LocalLLM
 import requests
 
+model = LocalLLM("codellama/CodeLlama-7b-Instruct-hf")
+
 st.write("\n")
 st.write("Welcome to the online reproducibility evaluation tool!")
 st.write("We follow guidelines provided by Simkó et al. (2022) (https://arxiv.org/abs/2210.11146)")
@@ -11,7 +14,7 @@ repo_link = st.text_input("Github repository link:", value="", type="default", h
 
 if (repo_link):
     verbose = 4 if checkbox else 3
-    evaluate(llm=None, verbose=verbose, repo_url=repo_link)
+    evaluate(llm=model, verbose=verbose, repo_url=repo_link)
 
 with st.form("my_form"):
     st.write("Notice something wrong? Please tell us so we can improve.")

evaluations/documentation.py

@@ -28,6 +28,7 @@ def evaluate(verbose, llm, zip, readme):
           package dependencies you need to install and how to train \
           and evaluate the proposed model? Please strictly \
           answer yes or no.\n\nA:'
+          llm.predict("HELP", prompt)
 
 
       manual_fail = False
@@ -47,7 +48,7 @@ def evaluate(verbose, llm, zip, readme):
           (len(re.findall("requirement", readme, re.IGNORECASE)) == 0)):
           log(verbose, "ERROR", "Readme file missing information about package dependencies")
           overall = "No"
-          
+
   return overall
 
 def count_comment_lines(lines):

evaluations/license.py

@@ -1,4 +1,4 @@
-from .utils import log, model_predict
+from .utils import log
 import re
 
 def evaluate(verbose, llm, zip, readme):
@@ -10,10 +10,9 @@ def evaluate(verbose, llm, zip, readme):
       ans = [row for row in license.split("\n") if row != ""]
 
       if (llm):
-          license = license[:50]
-          prompt = f"Q: {license}. This was an excerpt from a license \
-          file. Do you know the name of this license?"
-          ans = model_predict(prompt)
+          license = license
+          prompt = f"{license}. Please describe this type of license, what it allows and what it doesn't."
+          ans = llm.predict("HELP", prompt)
           log(verbose, "LOG", f"Found license: {ans}")
       else:
           log(verbose, "LOG", f"Found license file: {license_files[0]}")

evaluations/models.py

@@ -0,0 +1,47 @@
+from transformers import pipeline
+from huggingface_hub import InferenceClient
+import os
+
+system_messages = { "STRICT": "You are a chatbot evaluating github repositories, their python codes and corresponding readme files. Strictly answer the questions with Yes or No.",
+                    "HELP": "You are a chatbot evaluating github repositories, their python codes and corresponding readme files. Please help me answer the following question." }
+
+class LocalLLM():
+  def __init__(self, model_name):
+    self.pipe = pipeline("text-generation", model=model_name, max_new_tokens=1000, device_map={0: 0})
+
+  def predict(self, response_type, prompt):
+    messages = [
+        {"role": "system", "content": system_messages[response_type]},
+        {"role": "user", "content": prompt},
+    ]
+    res = self.pipe(messages)
+    res = res[0]["generated_text"]
+
+    res = [response for response in res if response["role"] == "assistant"][0]["content"]
+    res = res.strip()
+
+    return res
+
+class RemoteLLM():
+  def __init__(self):
+    token = os.getenv("hfToken")
+    API_URL = "https://api-inference.huggingface.co/models/openlm-research/open_llama_3b_v2"
+    headers = {"Authorization": f"Bearer {token}", "x-wait-for-model": "true"}
+
+    self.client = InferenceClient(
+        "meta-llama/Llama-3.1-8B-Instruct",
+        token=token,
+    )
+
+
+  def predict(self, response_type, prompt):
+    for message in self.client.chat_completion(
+        messages=[{"role": "system", "content": system_messages[response_type]}, 
+                  {"role": "user", "content": prompt}],
+        max_tokens=500,
+        stream=True,
+        ):
+        return message.choices[0].delta.content
+
+    return ""
+

evaluations/pitfalls.py

@@ -0,0 +1,6 @@
+from .utils import log, model_predict
+import re
+
+def evaluate(verbose, llm, zip, readme):
+  log(verbose, "TITLE", "\nLooking for common pitfalls...")
+  

evaluations/repo_evaluations.py

@@ -7,22 +7,11 @@ import os
 import numpy as np
 from huggingface_hub import InferenceClient
 
-API_URL = "https://api-inference.huggingface.co/models/openlm-research/open_llama_3b_v2"
-headers = {"Authorization": "Bearer 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", "x-wait-for-model": "true"}
-
-client = InferenceClient(
-    "meta-llama/Llama-3.1-8B-Instruct",
-    token="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",
-)
-
 def evaluate(llm, verbose, repo_url, title=None, year=None):
   repository_zip_name = "data/repo.zip"
   token = os.getenv("githubToken")
-  # token = userdata.get('githubToken')
 
-  if (llm):
-      init_llm(verbose)
-  else:
+  if (not(llm)):
       log(verbose, "LOG", "No LLM will be used for the evaluation.")
 
   results = { "pred_live": "Yes", "pred_dependencies": None, "pred_training": None, "pred_evaluation": None, "pred_weights": None, "pred_readme": None, "pred_license": None, "pred_stars": None, "pred_citations": None, "pred_valid": False}
@@ -54,9 +43,9 @@ def evaluate(llm, verbose, repo_url, title=None, year=None):
       readme = fetch_readme(zip)
       results["pred_stars"] = fetch_repo_stars(verbose, repo_url, token)
 
-
+      results["pred_license"] = license.evaluate(verbose, llm, zip, readme)
       if (len(zip.namelist()) <= 2):
-          log(verbose, "LOG", "Empty repository")
+          log(verbose, "LOG", "The repository is empty.")
           results["pred_live"] = "No"
           results["pred_training"] = "No"
           results["pred_evaluation"] = "No"
@@ -69,7 +58,6 @@ def evaluate(llm, verbose, repo_url, title=None, year=None):
           results["pred_weights"] = weights.evaluate(verbose, llm, zip, readme)
           results["pred_readme"] = documentation.evaluate(verbose, llm, zip, readme)
           results["pred_codetocomment"] = documentation.get_code_to_comment_ratio(zip)
-          results["pred_license"] = license.evaluate(verbose, llm, zip, readme)
 
       return results
   except Exception as e:
@@ -94,7 +82,7 @@ def full_evaluation():
       full_results.append(row)
   return pd.DataFrame(full_results)
 
-def midl_evaluations():
+def midl_evaluations(model):
   compare_to_gt = True
   paper_dump = pd.read_csv("data/dump.csv", sep="\t")
   verbose = 1
@@ -120,7 +108,7 @@ def midl_evaluations():
       print(f"\nEvaluating {idx+1} out of {len(paper_dump.index)} papers...")
       print(f'Paper title - "{row["title"]}" ({row["year"]})')
       print(f'Repository link - {row["url"]}')
-      result = evaluate(None, verbose, row["url"])
+      result = evaluate(model, verbose, row["url"])
       for column in result.keys():
           row[column] = result[column]
       full_results.append(row)

evaluations/utils.py

@@ -6,16 +6,6 @@ import json
 import streamlit as st
 
 
-def model_predict(client, prompt):
-    for message in client.chat_completion(
-        messages=[{"role": "system", "content": "You are a chatbot evaluating github repositories, their python codes and corresponding readme files. Strictly answer the questions with Yes or No."}, {"role": "user", "content": prompt}],
-        max_tokens=500,
-        stream=True,
-        ):
-        return message.choices[0].delta.content
-
-    return ""
-
 
 def get_api_link(url):
     username, repo_name = decompose_url(url)
@@ -72,10 +62,8 @@ def fetch_repo(verbose, repo_url, repo_name, token):
     if response.status_code == 200:
         with open(repo_name, 'wb') as file:
             file.write(response.content)
-
-        log(verbose, "LOG", "Repository downloaded successfully")
     if (response.status_code == 404):
-        log(verbose, "ERROR", "Repository private.")
+        log(verbose, "ERROR", "Repository private / Link broken.")
 
 def fetch_readme(zip):
     readme_files = [readme for readme in zip.namelist() if ((readme.endswith("README.MD") | readme.endswith("README.md") | readme.endswith("readme.md")) & (len(readme.split("/")) == 2))]

full_eval.py

@@ -8,4 +8,4 @@ load_dotenv()
 token = os.getenv("githubToken")
 
 res = full_evaluation()
-res.to_csv("results.csv", sep="\t", index=False)
+res.to_csv("data/results.csv", sep="\t", index=False)

midl.py

@@ -1,11 +1,13 @@
 from evaluations.repo_evaluations import midl_evaluations
-# importing os module for environment variables
+from evaluations.models import LocalLLM
 import os
-# importing necessary functions from dotenv library
-from dotenv import load_dotenv 
-# loading variables from .env file
+from dotenv import load_dotenv
 load_dotenv() 
 token = os.getenv("githubToken")
 
-res = midl_evaluations()
+
+# Load model directly
+
+model = LocalLLM("codellama/CodeLlama-7b-Instruct-hf")
+res = midl_evaluations(model)
 res.to_csv("results_midl.csv", sep="\t", index=False)