app.py

import spaces

import torch

print('torch version:', torch.__version__)

# import torch._dynamo
# torch._dynamo.config.suppress_errors = True
# torch._dynamo.disable()
# torch._dynamo.disallow_in_graph()

import gradio as gr

from transformers import AutoTokenizer, AutoModelForCausalLM

torch.set_float32_matmul_precision('high')

max_seq_length = 2048

tokenizer = AutoTokenizer.from_pretrained("ua-l/gemma-2-9b-legal-steps200-merged-16bit-uk")
model = AutoModelForCausalLM.from_pretrained(
    "ua-l/gemma-2-9b-legal-steps200-merged-16bit-uk",
    torch_dtype=torch.float16,
).to('cuda')
# compiled_model = torch.compile(model, mode="default")


print('Model dtype:', model.dtype)


@spaces.GPU
def predict(question):
    inputs = tokenizer(
    [f'''### Question:
    {question}
    
    ### Answer:
'''], return_tensors = "pt").to("cuda")

    outputs = model.generate(**inputs, max_new_tokens = 128)
    
    results = tokenizer.batch_decode(outputs, skip_special_tokens=True)

    return results[0]

inputs = gr.Textbox(lines=2, label="Enter a question", value="Як отримати виплати ВПО?")

outputs = gr.Textbox(label="Answer")

demo = gr.Interface(fn=predict, inputs=inputs, outputs=outputs)
demo.launch()