tokenizer can decode tensor for vllm test (#14)

- tokenizer can decode tensor for vllm test (da9637b13c17a1c828afd4c8d010df0fd081fd19)

tokenization_moonshot.py

@@ -16,6 +16,7 @@ from shutil import copyfile
 from tiktoken.load import load_tiktoken_bpe
 from tokenizers import AddedToken
 from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import to_py_obj
 from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
 
 
@@ -229,6 +230,8 @@ class TikTokenTokenizer(PreTrainedTokenizer):
         if len(kwargs) > 0:
             return super().decode(token_ids, **kwargs)
 
+        token_ids = to_py_obj(token_ids)
+
         if type(token_ids) is int:
             token_ids = [token_ids]