WIP

app.py

@@ -1,6 +1,7 @@
 import os, sys
 import tempfile
 import gradio as gr
+from app_tts import infer_tts
 from src.gradio_demo import SadTalker  
 # from src.utils.text2speech import TTSTalker
 from huggingface_hub import snapshot_download
@@ -39,6 +40,28 @@ def sadtalker_demo():
     # tts_talker = TTSTalker()
 
     with gr.Blocks(analytics_enabled=False) as sadtalker_interface:
+        gr.Markdown("""
+# 🎤 F5-TTS: Vietnamese Text-to-Speech Synthesis.
+# The model was trained with approximately 1000 hours of data on a RTX 3090 GPU. 
+Enter text and upload a sample voice to generate natural speech.
+""")
+        
+        with gr.Row():
+            ref_audio = gr.Audio(label="🔊 Sample Voice", type="filepath")
+            gen_text = gr.Textbox(label="📝 Text", placeholder="Enter the text to generate voice...", lines=3)
+        
+        speed = gr.Slider(0.3, 2.0, value=1.0, step=0.1, label="⚡ Speed")
+        btn_synthesize = gr.Button("🔥 Generate Voice")
+        
+        with gr.Row():
+            driven_audio = gr.Audio(label="🎧 Generated Audio", type="numpy")
+            output_spectrogram = gr.Image(label="📊 Spectrogram")
+        
+
+        btn_synthesize.click(infer_tts, inputs=[ref_audio, gen_text, speed], outputs=[driven_audio, output_spectrogram])
+
+        
+        
         gr.Markdown("<div align='center'> <h2> 😭 SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation (CVPR 2023) </span> </h2> \
                     <a style='font-size:18px;color: #efefef' href='https://arxiv.org/abs/2211.12194'>Arxiv</a> &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
                     <a style='font-size:18px;color: #efefef' href='https://sadtalker.github.io'>Homepage</a>  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
@@ -63,7 +86,7 @@ def sadtalker_demo():
                         gr.Markdown("Possible driving combinations: <br> 1. Audio only 2. Audio/IDLE Mode + Ref Video(pose, blink, pose+blink) 3. IDLE Mode only 4. Ref Video only (all) ")
 
                         with gr.Row():
-                            driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath", max_length=180) # 180s
+                            # driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath", max_length=180) # 180s
                             driven_audio_no = gr.Audio(label="Use IDLE mode, no audio is required", source="upload", type="filepath", visible=False)
 
                             with gr.Column():

app_tts.py

@@ -0,0 +1,107 @@
+import spaces
+import os
+from huggingface_hub import login
+import gradio as gr
+from cached_path import cached_path
+import tempfile
+from vinorm import TTSnorm
+
+from f5_tts.model import DiT
+from f5_tts.infer.utils_infer import (
+    preprocess_ref_audio_text,
+    load_vocoder,
+    load_model,
+    infer_process,
+    save_spectrogram,
+)
+
+# Retrieve token from secrets
+hf_token = os.getenv("HUGGINGFACEHUB_API_TOKEN")
+
+
+# Log in to Hugging Face
+if hf_token:
+    login(token=hf_token)
+
+def post_process(text):
+    text = " " + text + " "
+    text = text.replace(" . . ", " . ")
+    text = " " + text + " "
+    text = text.replace(" .. ", " . ")
+    text = " " + text + " "
+    text = text.replace(" , , ", " , ")
+    text = " " + text + " "
+    text = text.replace(" ,, ", " , ")
+    text = " " + text + " "
+    text = text.replace('"', "")
+    return " ".join(text.split())
+
+# Load models
+vocoder = load_vocoder()
+model = load_model(
+    DiT,
+    dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4),
+    ckpt_path=str(cached_path("hf://hynt/F5-TTS-Vietnamese-ViVoice/model_last.pt")),
+    vocab_file=str(cached_path("hf://hynt/F5-TTS-Vietnamese-ViVoice/config.json")),
+)
+
+@spaces.GPU
+def infer_tts(ref_audio_orig: str, gen_text: str, speed: float = 1.0, request: gr.Request = None):
+
+    if not ref_audio_orig:
+        raise gr.Error("Please upload a sample audio file.")
+    if not gen_text.strip():
+        raise gr.Error("Please enter the text content to generate voice.")
+    if len(gen_text.split()) > 1000:
+        raise gr.Error("Please enter text content with less than 1000 words.")
+    
+    try:
+        ref_audio, ref_text = preprocess_ref_audio_text(ref_audio_orig, "")
+        final_wave, final_sample_rate, spectrogram = infer_process(
+            ref_audio, ref_text.lower(), post_process(TTSnorm(gen_text)).lower(), model, vocoder, speed=speed
+        )
+        with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_spectrogram:
+            spectrogram_path = tmp_spectrogram.name
+            save_spectrogram(spectrogram, spectrogram_path)
+
+        return (final_sample_rate, final_wave), spectrogram_path
+    except Exception as e:
+        raise gr.Error(f"Error generating voice: {e}")
+
+# Gradio UI
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("""
+    # 🎤 F5-TTS: Vietnamese Text-to-Speech Synthesis.
+    # The model was trained with approximately 1000 hours of data on a RTX 3090 GPU. 
+    Enter text and upload a sample voice to generate natural speech.
+    """)
+    
+    with gr.Row():
+        ref_audio = gr.Audio(label="🔊 Sample Voice", type="filepath")
+        gen_text = gr.Textbox(label="📝 Text", placeholder="Enter the text to generate voice...", lines=3)
+    
+    speed = gr.Slider(0.3, 2.0, value=1.0, step=0.1, label="⚡ Speed")
+    btn_synthesize = gr.Button("🔥 Generate Voice")
+    
+    with gr.Row():
+        output_audio = gr.Audio(label="🎧 Generated Audio", type="numpy")
+        output_spectrogram = gr.Image(label="📊 Spectrogram")
+    
+    model_limitations = gr.Textbox(
+        value="""1. This model may not perform well with numerical characters, dates, special characters, etc. => A text normalization module is needed.
+2. The rhythm of some generated audios may be inconsistent or choppy => It is recommended to select clearly pronounced sample audios with minimal pauses for better synthesis quality.
+3. Default, reference audio text uses the pho-whisper-medium model, which may not always accurately recognize Vietnamese, resulting in poor voice synthesis quality.
+4. Inference with overly long paragraphs may produce poor results.""", 
+        label="❗ Model Limitations",
+        lines=4,
+        interactive=False
+    )
+
+    btn_synthesize.click(infer_tts, inputs=[ref_audio, gen_text, speed], outputs=[output_audio, output_spectrogram])
+
+# Run Gradio with share=True to get a gradio.live link
+# demo.queue().launch()
+
+if __name__ == "__main__":
+
+    demo.queue().launch()

requirements.txt

@@ -17,8 +17,29 @@ pyyaml
 joblib==1.1.0
 scikit-image==0.19.3
 basicsr==1.4.2
-facexlib==0.3.0
+facexlib
 dlib-bin
 gfpgan
 av
-safetensors
+safetensors
+gradio
+soundfile
+transformers
+bitsandbytes>0.37.0
+vinorm
+cached_path
+huggingface_hub
+accelerate>=0.33.0
+click
+datasets
+ema_pytorch>=0.5.2
+hydra-core>=1.3.0
+jieba
+matplotlib
+pypinyin
+tomli
+torchdiffeq
+transformers_stream_generator
+vocos
+wandb
+x_transformers>=1.31.14