Update app.py

app.py

@@ -1,3 +1,4 @@
+from matplotlib.pyplot import text
 import numpy as np
 import soundfile as sf
 import yaml
@@ -6,40 +7,86 @@ import tensorflow as tf
 
 from tensorflow_tts.inference import TFAutoModel
 from tensorflow_tts.inference import AutoProcessor
+from tensorflow_tts.inference import AutoConfig
 import gradio as gr
 
-# initialize fastspeech2 model.
-fastspeech2 = TFAutoModel.from_pretrained("tensorspeech/tts-fastspeech2-ljspeech-en")
+MODEL_NAMES = [
+    "Fastspeech2 + Melgan", 
+    "Tacotron2 + Melgan",
+]
 
-# initialize mb_melgan model
-mb_melgan = TFAutoModel.from_pretrained("tensorspeech/tts-mb_melgan-ljspeech-en")
+fastspeech = TFAutoModel.from_pretrained("tensorspeech/tts-fastspeech-ljspeech-en", name="fastspeech")
+fastspeech2 = TFAutoModel.from_pretrained("tensorspeech/tts-fastspeech2-ljspeech-en", name="fastspeech2")
+tacotron2 = TFAutoModel.from_pretrained("tensorspeech/tts-tacotron2-ljspeech-en", name="tacotron2")
+melgan = TFAutoModel.from_pretrained("tensorspeech/tts-melgan-ljspeech-en", name="melgan")
+mb_melgan = TFAutoModel.from_pretrained("tensorspeech/tts-mb_melgan-ljspeech-en", name="mb_melgan")
+melgan_stft_config = AutoConfig.from_pretrained('TensorFlowTTS/examples/melgan_stft/conf/melgan_stft.v1.yaml')
+melgan_stft = TFAutoModel.from_pretrained(
+    config=melgan_stft_config,
+    pretrained_path="melgan.stft-2M.h5",
+    name="melgan_stft"
+)
 
+MODEL_DICT = {
+    "Fastspeech2" : fastspeech2,
+    "Tacotron2" : tacotron2,
+    "Melgan": melgan,
+    "MB-Melgan": mb_melgan,
+    "Melgan-STFT": melgan_stft
+}
 
-# inference
-processor = AutoProcessor.from_pretrained("tensorspeech/tts-fastspeech2-ljspeech-en")
+def inference(input):
+    input_text, model_type = input[0], input[1]
 
-def inference(text):
-  input_ids = processor.text_to_sequence(text)
-  # fastspeech inference
-  
-  mel_before, mel_after, duration_outputs, _, _ = fastspeech2.inference(
-      input_ids=tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
-      speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
-      speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
-      f0_ratios =tf.convert_to_tensor([1.0], dtype=tf.float32),
-      energy_ratios =tf.convert_to_tensor([1.0], dtype=tf.float32),
-  )
-  
-  # melgan inference
-  audio_before = mb_melgan.inference(mel_before)[0, :, 0]
-  audio_after = mb_melgan.inference(mel_after)[0, :, 0]
-  
-  # save to file
-  sf.write('./audio_before.wav', audio_before, 22050, "PCM_16")
-  sf.write('./audio_after.wav', audio_after, 22050, "PCM_16")
-  return './audio_after.wav'
+    text2mel_name, vocoder_name = model_type.split(" + ")
+    text2mel_model, vocoder_model = MODEL_DICT[text2mel_name], MODEL_DICT[vocoder_name]
+    processor = AutoProcessor.from_pretrained(text2mel_name)
+    input_ids = processor.text_to_sequence(input_text)
+    
+    if text2mel_name == "Tacotron":
+        _, mel_outputs, stop_token_prediction, alignment_history = text2mel_model.inference(
+            tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
+            tf.convert_to_tensor([len(input_ids)], tf.int32),
+            tf.convert_to_tensor([0], dtype=tf.int32)
+        )
+    elif text2mel_name == "Fastspeech":
+        mel_before, mel_outputs, duration_outputs = text2mel_model.inference(
+            input_ids=tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
+            speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
+            speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+        )
+    elif text2mel_name == "Fastspeech2":
+        mel_before, mel_outputs, duration_outputs, _, _ = text2mel_model.inference(
+            tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
+            speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
+            speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+            f0_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+            energy_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+        )
+    else:
+        raise ValueError("Only TACOTRON, FASTSPEECH, FASTSPEECH2 are supported on text2mel_name")
+
+    # vocoder part
+    if vocoder_name == "Melgan" or vocoder_name == "Melgan-STFT":
+        audio = vocoder_model(mel_outputs)[0, :, 0]
+    elif vocoder_name == "MB-Melgan":
+        audio = vocoder_model(mel_outputs)[0, :, 0]
+    else:
+        raise ValueError("Only MELGAN, MELGAN-STFT and MB_MELGAN are supported on vocoder_name")
+
+    # if text2mel_name == "TACOTRON":
+    #     return mel_outputs.numpy(), alignment_history.numpy(), audio.numpy()
+    # else:
+    #     return mel_outputs.numpy(), audio.numpy()
+
+    sf.write('./audio_after.wav', audio, 22050, "PCM_16")
+    return './audio_after.wav'
   
-inputs = gr.inputs.Textbox(lines=5, label="Input Text")
+inputs = [
+    gr.inputs.Textbox(lines=5, label="Input Text"),
+    gr.inputs.Radio(label="Pick a TTS Model",choices=MODEL_NAMES,)
+]
+
 outputs =  gr.outputs.Audio(type="file", label="Output Audio")
 
 
@@ -48,8 +95,8 @@ description = "Gradio demo for TensorFlowTTS: Real-Time State-of-the-art Speech
 article = "<p style='text-align: center'><a href='https://tensorspeech.github.io/TensorFlowTTS/'>TensorFlowTTS: Real-Time State-of-the-art Speech Synthesis for Tensorflow 2</a> | <a href='https://github.com/TensorSpeech/TensorFlowTTS'>Github Repo</a></p>"
 
 examples = [
- ["TensorFlowTTS provides real-time state-of-the-art speech synthesis architectures such as Tacotron-2, Melgan, Multiband-Melgan, FastSpeech, FastSpeech2 based-on TensorFlow 2."],
- ["With Tensorflow 2, we can speed-up training/inference progress, optimizer further by using fake-quantize aware and pruning, make TTS models can be run faster than real-time and be able to deploy on mobile devices or embedded systems."]   
+    ["TensorFlowTTS provides real-time state-of-the-art speech synthesis architectures such as Tacotron-2, Melgan, Multiband-Melgan, FastSpeech, FastSpeech2 based-on TensorFlow 2."],
+    ["With Tensorflow 2, we can speed-up training/inference progress, optimizer further by using fake-quantize aware and pruning, make TTS models can be run faster than real-time and be able to deploy on mobile devices or embedded systems."]   
 ]
 
 gr.Interface(inference, inputs, outputs, title=title, description=description, article=article, examples=examples).launch()