File size: 1,423 Bytes
1be71e1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 |
from ctransformers import AutoModelForCausalLM
import gradio as gr
def generate_prompt(history):
prompt = start_message
for chain in history[:-1]:
prompt += f"<|im_start|>user\n{chain[0]}<|im_end|>" +\
f"<|im_start|>assistant\n{chain[1]}<|im_end|>"
prompt += f"<|im_start|>user\n{history[-1][0]}<|im_end|>" +\
"<|im_start|>assistant\n"
return prompt
def generate(history):
prompt = generate_prompt(history)
streamer = llm(prompt, stop = stop_tokens, stream=True, threads=2)
return streamer
llm = AutoModelForCausalLM.from_pretrained("model/ggml-model-q8_0.bin", model_type='mpt')
stop_tokens = ["<|im_end|>", "<|endoftext|>"]
start_message = """<|im_start|>system
You are a helpful assistant chatbot.<|im_end|>
"""
with gr.Blocks() as demo:
chatbot = gr.Chatbot()
msg = gr.Textbox()
clear = gr.Button("Clear")
def user(user_message, history):
return "", history + [[user_message, ""]]
def bot(history):
streamer = generate(history)
for token in streamer:
history[-1][1] += token
yield history
msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
bot, chatbot, chatbot
)
clear.click(lambda: None, None, chatbot, queue=False)
demo.queue()
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860) |