import spaces
import argparse
import os
import shutil
import cv2
import gradio as gr
import numpy as np
import torch
from facexlib.utils.face_restoration_helper import FaceRestoreHelper
import huggingface_hub
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision.transforms.functional import normalize
from dreamo.dreamo_pipeline import DreamOPipeline
from dreamo.utils import img2tensor, resize_numpy_image_area, tensor2img, resize_numpy_image_long
from tools import BEN2
parser = argparse.ArgumentParser()
parser.add_argument('--port', type=int, default=8080)
parser.add_argument('--no_turbo', action='store_true')
args = parser.parse_args()
huggingface_hub.login(os.getenv('HF_TOKEN'))
try:
shutil.rmtree('gradio_cached_examples')
except FileNotFoundError:
print("cache folder not exist")
class Generator:
def __init__(self):
device = torch.device('cuda')
# preprocessing models
# background remove model: BEN2
self.bg_rm_model = BEN2.BEN_Base().to(device).eval()
hf_hub_download(repo_id='PramaLLC/BEN2', filename='BEN2_Base.pth', local_dir='models')
self.bg_rm_model.loadcheckpoints('models/BEN2_Base.pth')
# face crop and align tool: facexlib
self.face_helper = FaceRestoreHelper(
upscale_factor=1,
face_size=512,
crop_ratio=(1, 1),
det_model='retinaface_resnet50',
save_ext='png',
device=device,
)
# load dreamo
model_root = 'black-forest-labs/FLUX.1-dev'
dreamo_pipeline = DreamOPipeline.from_pretrained(model_root, torch_dtype=torch.bfloat16)
dreamo_pipeline.load_dreamo_model(device, use_turbo=not args.no_turbo)
self.dreamo_pipeline = dreamo_pipeline.to(device)
@torch.no_grad()
def get_align_face(self, img):
# the face preprocessing code is same as PuLID
self.face_helper.clean_all()
image_bgr = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
self.face_helper.read_image(image_bgr)
self.face_helper.get_face_landmarks_5(only_center_face=True)
self.face_helper.align_warp_face()
if len(self.face_helper.cropped_faces) == 0:
return None
align_face = self.face_helper.cropped_faces[0]
input = img2tensor(align_face, bgr2rgb=True).unsqueeze(0) / 255.0
input = input.to(torch.device("cuda"))
parsing_out = self.face_helper.face_parse(normalize(input, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]))[0]
parsing_out = parsing_out.argmax(dim=1, keepdim=True)
bg_label = [0, 16, 18, 7, 8, 9, 14, 15]
bg = sum(parsing_out == i for i in bg_label).bool()
white_image = torch.ones_like(input)
# only keep the face features
face_features_image = torch.where(bg, white_image, input)
face_features_image = tensor2img(face_features_image, rgb2bgr=False)
return face_features_image
generator = Generator()
@spaces.GPU
@torch.inference_mode()
def generate_image(
ref_image1,
ref_image2,
ref_task1,
ref_task2,
prompt,
seed,
width=1024,
height=1024,
ref_res=512,
num_steps=12,
guidance=3.5,
true_cfg=1,
cfg_start_step=0,
cfg_end_step=0,
neg_prompt='',
neg_guidance=3.5,
first_step_guidance=0,
):
print(prompt)
ref_conds = []
debug_images = []
ref_images = [ref_image1, ref_image2]
ref_tasks = [ref_task1, ref_task2]
for idx, (ref_image, ref_task) in enumerate(zip(ref_images, ref_tasks)):
if ref_image is not None:
if ref_task == "id":
ref_image = resize_numpy_image_long(ref_image, 1024)
ref_image = generator.get_align_face(ref_image)
elif ref_task != "style":
ref_image = generator.bg_rm_model.inference(Image.fromarray(ref_image))
if ref_task != "id":
ref_image = resize_numpy_image_area(np.array(ref_image), ref_res * ref_res)
debug_images.append(ref_image)
ref_image = img2tensor(ref_image, bgr2rgb=False).unsqueeze(0) / 255.0
ref_image = 2 * ref_image - 1.0
ref_conds.append(
{
'img': ref_image,
'task': ref_task,
'idx': idx + 1,
}
)
seed = int(seed)
if seed == -1:
seed = torch.Generator(device="cpu").seed()
image = generator.dreamo_pipeline(
prompt=prompt,
width=width,
height=height,
num_inference_steps=num_steps,
guidance_scale=guidance,
ref_conds=ref_conds,
generator=torch.Generator(device="cpu").manual_seed(seed),
true_cfg_scale=true_cfg,
true_cfg_start_step=cfg_start_step,
true_cfg_end_step=cfg_end_step,
negative_prompt=neg_prompt,
neg_guidance_scale=neg_guidance,
first_step_guidance_scale=first_step_guidance if first_step_guidance > 0 else guidance,
).images[0]
return image, debug_images, seed
_HEADER_ = '''
🚩 Update Notes:
- 2025.05.11: We have updated the model to mitigate over-saturation and plastic-face issues. The new version shows consistent improvements over the previous release.
- 2025.05.13: 'DreamO Video' Integration version Release
'''
_CITE_ = r"""
If DreamO is helpful, please help to ⭐ the community. Thanks!
---
📧 **Contact**
If you have any questions or feedbacks, feel free to open a discussion or contact arxivgpt@gmail.com
"""
def create_demo():
with gr.Blocks(theme="apriel") as demo:
gr.Markdown(_HEADER_)
with gr.Row():
with gr.Column():
with gr.Row():
ref_image1 = gr.Image(label="ref image 1", type="numpy", height=256)
ref_image2 = gr.Image(label="ref image 2", type="numpy", height=256)
with gr.Row():
ref_task1 = gr.Dropdown(choices=["ip", "id", "style"], value="ip", label="task for ref image 1")
ref_task2 = gr.Dropdown(choices=["ip", "id", "style"], value="ip", label="task for ref image 2")
prompt = gr.Textbox(label="Prompt", value="a person playing guitar in the street")
width = gr.Slider(768, 1024, 1024, step=16, label="Width")
height = gr.Slider(768, 1024, 1024, step=16, label="Height")
num_steps = gr.Slider(8, 30, 12, step=1, label="Number of steps")
guidance = gr.Slider(1.0, 10.0, 3.5, step=0.1, label="Guidance")
seed = gr.Textbox(label="Seed (-1 for random)", value="-1")
with gr.Accordion("Advanced Options", open=False, visible=False):
ref_res = gr.Slider(512, 1024, 512, step=16, label="resolution for ref image")
neg_prompt = gr.Textbox(label="Neg Prompt", value="")
neg_guidance = gr.Slider(1.0, 10.0, 3.5, step=0.1, label="Neg Guidance")
true_cfg = gr.Slider(1, 5, 1, step=0.1, label="true cfg")
cfg_start_step = gr.Slider(0, 30, 0, step=1, label="cfg start step")
cfg_end_step = gr.Slider(0, 30, 0, step=1, label="cfg end step")
first_step_guidance = gr.Slider(0, 10, 0, step=0.1, label="first step guidance")
generate_btn = gr.Button("Generate")
gr.Markdown(_CITE_)
with gr.Column():
output_image = gr.Image(label="Generated Image", format='png')
debug_image = gr.Gallery(
label="Preprocessing output (including possible face crop and background remove)",
elem_id="gallery",
)
seed_output = gr.Textbox(label="Used Seed")
with gr.Row(), gr.Column():
gr.Markdown("## Examples")
example_inps = [
[
'example_inputs/woman1.png',
None,
'ip',
'ip',
'profile shot dark photo of a 25-year-old female with smoke escaping from her mouth, the backlit smoke gives the image an ephemeral quality, natural face, natural eyebrows, natural skin texture, award winning photo, highly detailed face, atmospheric lighting, film grain, monochrome', # noqa E501
9180879731249039735,
],
[
'example_inputs/man1.png',
None,
'ip',
'ip',
'a man sitting on the cloud, playing guitar',
1206523688721442817,
],
[
'example_inputs/toy1.png',
None,
'ip',
'ip',
'a purple toy holding a sign saying "DreamO", on the mountain',
10441727852953907380,
],
[
'example_inputs/perfume.png',
None,
'ip',
'ip',
'a perfume under spotlight',
116150031980664704,
],
[
'example_inputs/hinton.jpeg',
None,
'id',
'ip',
'portrait, Chibi',
5443415087540486371,
],
[
'example_inputs/mickey.png',
None,
'style',
'ip',
'generate a same style image. A rooster wearing overalls.',
6245580464677124951,
],
[
'example_inputs/mountain.png',
None,
'style',
'ip',
'generate a same style image. A pavilion by the river, and the distant mountains are endless',
5248066378927500767,
],
[
'example_inputs/shirt.png',
'example_inputs/skirt.jpeg',
'ip',
'ip',
'A girl is wearing a short-sleeved shirt and a short skirt on the beach.',
9514069256241143615,
],
[
'example_inputs/woman2.png',
'example_inputs/dress.png',
'id',
'ip',
'the woman wearing a dress, In the banquet hall',
7698454872441022867,
],
[
'example_inputs/dog1.png',
'example_inputs/dog2.png',
'ip',
'ip',
'two dogs in the jungle',
6187006025405083344,
],
[
'example_inputs/woman3.png',
'example_inputs/cat.png',
'ip',
'ip',
'A girl rides a giant cat, walking in the noisy modern city. High definition, realistic, non-cartoonish. Excellent photography work, 8k high definition.', # noqa E501
11980469406460273604,
],
[
'example_inputs/man2.jpeg',
'example_inputs/woman4.jpeg',
'ip',
'ip',
'a man is dancing with a woman in the room',
8303780338601106219,
],
]
gr.Examples(
examples=example_inps,
inputs=[ref_image1, ref_image2, ref_task1, ref_task2, prompt, seed],
label='row 1-4: IP task; row 5: ID task; row 6-7: Style task. row 8-9: Try-On task; row 10-12: Multi IP',
cache_examples='lazy',
outputs=[output_image, debug_image, seed_output],
fn=generate_image,
)
generate_btn.click(
fn=generate_image,
inputs=[
ref_image1,
ref_image2,
ref_task1,
ref_task2,
prompt,
seed,
width,
height,
ref_res,
num_steps,
guidance,
true_cfg,
cfg_start_step,
cfg_end_step,
neg_prompt,
neg_guidance,
first_step_guidance,
],
outputs=[output_image, debug_image, seed_output],
)
return demo
if __name__ == '__main__':
demo = create_demo()
demo.launch()