gradio infer

F5-TTS/src/f5_tts/infer/infer_cli_test.py

@@ -21,7 +21,7 @@ from f5_tts.infer.utils_infer import (
     mel_spec_type,
     target_rms,
     cross_fade_duration,
-    nfe_step,
+    #nfe_step,
     cfg_strength,
     sway_sampling_coef,
     speed,
@@ -68,7 +68,7 @@ parser.add_argument(
     "--ckpt_file",
     type=str,
     help="The path to model checkpoint .pt, leave blank to use default",
-    default="",
+    default="./F5-TTS/ckpts/v2c/v2c_s44.pt",
 )
 parser.add_argument(
     "-v",
@@ -143,11 +143,11 @@ parser.add_argument(
     type=float,
     help=f"Duration of cross-fade between audio segments in seconds, default {cross_fade_duration}",
 )
-parser.add_argument(
-    "--nfe_step",
-    type=int,
-    help=f"The number of function evaluation (denoising steps), default {nfe_step}",
-)
+#parser.add_argument(
+#    "--nfe_step",
+#    type=int,
+#    help=f"The number of function evaluation (denoising steps), default {nfe_step}",
+#)
 parser.add_argument(
     "--cfg_strength",
     type=float,
@@ -177,7 +177,7 @@ parser.add_argument(
 parser.add_argument(
     "--end",
     type=int,
-    default=99999999,
+    default=1,
 )
 parser.add_argument(
     "--v2a_path",
@@ -239,7 +239,7 @@ ref_text = (
 gen_text = args.gen_text or config.get("gen_text", "Here we generate something just for test.")
 gen_file = args.gen_file or config.get("gen_file", "")
 
-output_dir = args.output_dir or config.get("output_dir", "tests")
+#output_dir = args.output_dir or config.get("output_dir", "tests")
 output_file = args.output_file or config.get(
     "output_file", f"infer_cli_{datetime.now().strftime(r'%Y%m%d_%H%M%S')}.wav"
 )
@@ -251,13 +251,13 @@ load_vocoder_from_local = args.load_vocoder_from_local or config.get("load_vocod
 vocoder_name = args.vocoder_name or config.get("vocoder_name", mel_spec_type)
 target_rms = args.target_rms or config.get("target_rms", target_rms)
 cross_fade_duration = args.cross_fade_duration or config.get("cross_fade_duration", cross_fade_duration)
-nfe_step = args.nfe_step or config.get("nfe_step", nfe_step)
+#nfe_step = args.nfe_step or config.get("nfe_step", nfe_step)
 cfg_strength = args.cfg_strength or config.get("cfg_strength", cfg_strength)
 sway_sampling_coef = args.sway_sampling_coef or config.get("sway_sampling_coef", sway_sampling_coef)
 speed = args.speed or config.get("speed", speed)
 fix_duration = args.fix_duration or config.get("fix_duration", fix_duration)
 
-print("############nfe_step", nfe_step, vocoder_name)
+#print("############nfe_step", nfe_step, vocoder_name)
 
 
 # patches for pip pkg user
@@ -280,12 +280,12 @@ if gen_file:
 
 # output path
 
-wave_path = Path(output_dir) / output_file
-# spectrogram_path = Path(output_dir) / "infer_cli_out.png"
-if save_chunk:
-    output_chunk_dir = os.path.join(output_dir, f"{Path(output_file).stem}_chunks")
-    if not os.path.exists(output_chunk_dir):
-        os.makedirs(output_chunk_dir)
+#wave_path = Path(output_dir) / output_file
+## spectrogram_path = Path(output_dir) / "infer_cli_out.png"
+#if save_chunk:
+#    output_chunk_dir = os.path.join(output_dir, f"{Path(output_file).stem}_chunks")
+#    if not os.path.exists(output_chunk_dir):
+#        os.makedirs(output_chunk_dir)
 
 
 # load vocoder
@@ -335,7 +335,7 @@ ema_model = load_model(model_cls, model_cfg, ckpt_file, mel_spec_type=vocoder_na
 # inference process
 
 
-def main(ref_audio, ref_text, gen_text, energy):
+def main(ref_audio, ref_text, gen_text, energy, nfe_step):
     main_voice = {"ref_audio": ref_audio, "ref_text": ref_text}
     if "voices" not in config:
         voices = {"main": main_voice}
@@ -431,9 +431,14 @@ def normalize_wav(waveform, waveform_ref):
     return waveform
 
 
-if __name__ == "__main__":
-    
-    v2a_path = args.v2a_path
+#if __name__ == "__main__":
+def v2s_infer(output_dir, v2a_path, wav_p, txt_p, video, v2a_wav, txt, nfe_step):
+    #v2a_path = args.v2a_path
+    args.wav_p = wav_p
+    args.txt_p = txt_p
+    args.video = video
+    args.v2a_wav = v2a_wav
+    args.txt = txt
     
     if args.wav_p == "":
         scp = args.infer_list
@@ -493,7 +498,7 @@ if __name__ == "__main__":
                 ####wav_gen, sr_gen = main(wav_p, txt_p, txt, [torch.zeros_like(energy_p), torch.zeros_like(energy)])
                 ####wav_gen, sr_gen = main(wav_p, txt_p, txt, None)
                 ####wav_gen, sr_gen = main(wav, txt, txt, None)
-                wav_gen, sr_gen = main(wav_p, txt_p, txt, [energy_p, energy])
+                wav_gen, sr_gen = main(wav_p, txt_p, txt, [energy_p, energy], nfe_step)
                 ####wav_gen, sr_gen = main(wav, txt, txt, [energy.clone(), energy])
                 wav_gen = torch.from_numpy(wav_gen).unsqueeze(0)
                 assert(sr_gen == 24000)

MMAudio/demo.py

@@ -29,16 +29,16 @@ log = logging.getLogger()
 
 
 @torch.inference_mode()
-def main():
+def v2a_load():
     setup_eval_logging()
 
     parser = ArgumentParser()
     parser.add_argument('--variant',
                         type=str,
-                        default='large_44k',
+                        #default='large_44k',
                         #default='small_16k',
                         #default='medium_44k',
-                        #default='small_44k',
+                        default='small_44k',
                         help='small_16k, small_44k, medium_44k, large_44k, large_44k_v2')
     parser.add_argument('--video', type=Path, help='Path to the video file')
     parser.add_argument('--prompt', type=str, help='Input prompt', default='')
@@ -64,23 +64,23 @@ def main():
     if args.variant not in all_model_cfg:
         raise ValueError(f'Unknown model variant: {args.variant}')
     model: ModelConfig = all_model_cfg[args.variant]
-    model.download_if_needed()
+    #model.download_if_needed()
     seq_cfg = model.seq_cfg
 
-    if args.video:
-        #video_path: Path = Path(args.video).expanduser()
-        video_path = args.video
-    else:
-        video_path = None
-    prompt: str = args.prompt
-    negative_prompt: str = args.negative_prompt
-    output_dir: str = args.output.expanduser()
+    #if args.video:
+    #    #video_path: Path = Path(args.video).expanduser()
+    #    video_path = args.video
+    #else:
+    #    video_path = None
+    #prompt: str = args.prompt
+    #negative_prompt: str = args.negative_prompt
+    #output_dir: str = args.output.expanduser()
     seed: int = args.seed
-    num_steps: int = args.num_steps
+    #num_steps: int = args.num_steps
     duration: float = args.duration
     cfg_strength: float = args.cfg_strength
     skip_video_composite: bool = args.skip_video_composite
-    mask_away_clip: bool = args.mask_away_clip
+    #mask_away_clip: bool = args.mask_away_clip
 
     device = 'cpu'
     if torch.cuda.is_available():
@@ -92,19 +92,26 @@ def main():
     print("full_precision", args.full_precision)
     dtype = torch.float32 if args.full_precision else torch.bfloat16
 
-    output_dir.mkdir(parents=True, exist_ok=True)
+    #output_dir.mkdir(parents=True, exist_ok=True)
 
     # load a pretrained model
     net: MMAudio = get_my_mmaudio(model.model_name).to(device, dtype).eval()
     ####model.model_path = "/ailab-train/speech/zhanghaomin/codes3/MMAudio-main/output/exp_1/exp_1_shadow.pth"
+    model.model_path = "MMAudio" / model.model_path
+    print("model.model_path", model.model_path)
     net.load_weights(torch.load(model.model_path, map_location=device, weights_only=True))
     log.info(f'Loaded weights from {model.model_path}')
 
     # misc setup
     rng = torch.Generator(device=device)
     rng.manual_seed(seed)
-    fm = FlowMatching(min_sigma=0, inference_mode='euler', num_steps=num_steps)
+    #fm = FlowMatching(min_sigma=0, inference_mode='euler', num_steps=num_steps)
 
+    model.vae_path = "MMAudio" / model.vae_path
+    model.synchformer_ckpt = "MMAudio" / model.synchformer_ckpt
+    print("model.vae_path", model.vae_path)
+    print("model.synchformer_ckpt", model.synchformer_ckpt)
+    print("model.bigvgan_16k_path", model.bigvgan_16k_path)
     feature_utils = FeaturesUtils(tod_vae_ckpt=model.vae_path,
                                   synchformer_ckpt=model.synchformer_ckpt,
                                   enable_conditions=True,
@@ -112,7 +119,19 @@ def main():
                                   bigvgan_vocoder_ckpt=model.bigvgan_16k_path,
                                   need_vae_encoder=False)
     feature_utils = feature_utils.to(device, dtype).eval()
+    return net, seq_cfg, rng, feature_utils, args
+
 
+@torch.inference_mode()
+def v2a_infer(output_dir, video_path, prompt, num_steps, loaded):
+    net, seq_cfg, rng, feature_utils, args = loaded
+    negative_prompt = ""
+    duration = args.duration
+    cfg_strength = args.cfg_strength
+    skip_video_composite = args.skip_video_composite
+    mask_away_clip = args.mask_away_clip
+    fm = FlowMatching(min_sigma=0, inference_mode='euler', num_steps=num_steps)
+    
     ####test_scp = "/ailab-train/speech/zhanghaomin/animation_dataset_v2a/test.scp"
     #test_scp = "/ailab-train/speech/zhanghaomin/datas/v2cdata/tmp.scp"
     #test_scp = "/ailab-train/speech/zhanghaomin/datas/v2cdata/test.scp"

app.py

@@ -22,18 +22,31 @@ import numpy as np
 
 from huggingface_hub import hf_hub_download
 
-model_path = "./F5-TTS/ckpts/v2c/"
+if True:
+    model_path = "./F5-TTS/ckpts/v2c/"
 
-if not os.path.exists(model_path):
-    os.makedirs(model_path)
+    if not os.path.exists(model_path):
+        os.makedirs(model_path)
 
-file_path = hf_hub_download(repo_id="lshzhm/DeepAudio-V1", filename="v2c_s44.pt", local_dir=model_path)
+    file_path = hf_hub_download(repo_id="lshzhm/DeepAudio-V1", filename="v2c_s44.pt", local_dir=model_path)
 
-print(f"Model saved at: {file_path}")
+    print(f"Model saved at: {file_path}")
 
 log = logging.getLogger()
 
 
+import sys
+sys.path.insert(0, "./F5-TTS/src/")
+from f5_tts.infer.infer_cli_test import v2s_infer
+
+
+import sys
+sys.path.insert(0, "./MMAudio/")
+from demo import v2a_load, v2a_infer
+
+v2a_loaded = v2a_load()
+
+
 #@spaces.GPU(duration=120)
 def video_to_audio_and_speech(video: gr.Video, prompt: str, v2a_num_steps: int, text: str, audio_prompt: gr.Audio, text_prompt: str, v2s_num_steps: int):
 
@@ -64,18 +77,26 @@ def video_to_audio_and_speech(video: gr.Video, prompt: str, v2a_num_steps: int,
     else:
         shutil.copy(audio_prompt, audio_p_path)
     
-    if prompt == "":
-        command = "cd ./MMAudio; python ./demo.py --variant small_44k --output %s --video %s --calc_energy 1 --num_steps %d" % (output_dir, video_path, v2a_num_steps)
-    else:
-        command = "cd ./MMAudio; python ./demo.py --variant small_44k --output %s --video %s --prompt %s --calc_energy 1 --num_steps %d" % (output_dir, video_path, prompt, v2a_num_steps)
-    print("v2a command", command)
-    os.system(command)
+    #if prompt == "":
+    #    command = "cd ./MMAudio; python ./demo.py --variant small_44k --output %s --video %s --calc_energy 1 --num_steps %d" % (output_dir, video_path, v2a_num_steps)
+    #else:
+    #    command = "cd ./MMAudio; python ./demo.py --variant small_44k --output %s --video %s --prompt %s --calc_energy 1 --num_steps %d" % (output_dir, video_path, prompt, v2a_num_steps)
+    #print("v2a command", command)
+    #os.system(command)
+    
+    
+    v2a_infer(output_dir, video_path, prompt, v2a_num_steps, v2a_loaded)
+    
     
     video_gen = video_save_path[:-4]+".mp4.gen.mp4"
     
-    command = "python ./F5-TTS/src/f5_tts/infer/infer_cli_test.py --output_dir %s --start 0 --end 1 --ckpt_file ./F5-TTS/ckpts/v2c/v2c_s44.pt --v2a_path %s --wav_p %s --txt_p \"%s\" --video %s --v2a_wav %s --txt \"%s\" --nfe_step %d" % (output_dir, output_dir, audio_p_path, text_prompt, video_save_path, video_save_path[:-4]+".flac", text, v2s_num_steps)
-    print("v2s command", command, video_gen)
-    os.system(command)
+    #command = "python ./F5-TTS/src/f5_tts/infer/infer_cli_test.py --output_dir %s --start 0 --end 1 --ckpt_file ./F5-TTS/ckpts/v2c/v2c_s44.pt --v2a_path %s --wav_p %s --txt_p \"%s\" --video %s --v2a_wav %s --txt \"%s\" --nfe_step %d" % (output_dir, output_dir, audio_p_path, text_prompt, video_save_path, video_save_path[:-4]+".flac", text, v2s_num_steps)
+    #print("v2s command", command, video_gen)
+    #os.system(command)
+    
+    
+    v2s_infer(output_dir, output_dir, audio_p_path, text_prompt, video_save_path, video_save_path[:-4]+".flac", text, v2s_num_steps)
+    
     
     return video_save_path, video_gen