Update app.py

app.py

@@ -27,10 +27,12 @@ initial_list_of_models = [
     "CohereForAI/c4ai-command-r-v01",
     "CohereForAI/c4ai-command-r-plus",
     "CohereForAI/aya-101",
+    "aubmindlab/bert-base-arabertv02-twitter",
     "aubmindlab/bert-base-arabertv02",
     "aubmindlab/bert-base-arabertv2",
     "aubmindlab/bert-base-arabertv01",
-    "aubmindlab/bert-base-arabert"
+    "aubmindlab/bert-base-arabert",
+    "aubmindlab/aragpt2-mega"
 ]
 
 dataframe_path = Path(__file__).parent / "arabic_tokenizers_leaderboard.jsonl"
@@ -63,7 +65,7 @@ def benchmark_tokenizer(model_name) -> float:
         model_name, use_fast=True, trust_remote_code=True
     )
     vocab_size = tokenizer.vocab_size
-    if "arabert" in model_name:
+    if "arabert" in model_name or "aragpt2" in model_name:
         arabert_prep = ArabertPreprocessor(model_name=model_name)
         arabic_dataset1_preped = [ arabert_prep.preprocess(x) for x in tqdm(arabic_dataset1, desc="Arabert Prep Dataset 1")]
         arabic_dataset2_preped = [ arabert_prep.preprocess(x) for x in tqdm(arabic_dataset2, desc="Arabert Prep Dataset 2")]
@@ -192,7 +194,7 @@ def decode_bpe_tokens(tokens):
 
 def tokenize_text(text, chosen_model, better_tokenization=False):
     tokenizer = AutoTokenizer.from_pretrained(chosen_model)
-    if "arabert" in chosen_model:
+    if "arabert" in chosen_model or "aragpt2" in chosen_model:
         arabert_prep = ArabertPreprocessor(model_name=chosen_model)
         text = arabert_prep.preprocess(text)
     tokenized_text = decode_bpe_tokens(tokenizer.tokenize(text))