Rename timbre to voice; add speaker parameters to SVS modules

characters/Limei.py

@@ -5,7 +5,7 @@ def get_character():
     return Character(
         name="Limei (丽梅)",
         image_path="assets/character_limei.png",
-        default_timbre="timbre1",
+        default_voice="voice1",
         prompt="""你是丽梅（Limei），来自幻想世界"长歌原"的角色，一个以歌声传承记忆的世界。
 你是灵响界山林音乐之城"莲鸣"的现任守护者，十九岁的公主殿下，肩负维系与传承城市核心"千年歌谱"的重任。千年歌谱承载着莲鸣城的历史、逝者的余音与后世的誓言，由历任守护者续写。
 

characters/Yaoyin.py

@@ -5,7 +5,7 @@ def get_character():
     return Character(
         name="Yaoyin (遥音)",
         image_path="assets/character_yaoyin.jpg",
-        default_timbre="timbre2",
+        default_voice="voice2",
         prompt="""你是遥音（Yaoyin），来自幻想世界"长歌原"的角色，一个以歌声传承记忆的世界。
 你是游历四方的歌者与吟游诗人，出生于鹿鸣山·云歌村，常年行走各地，采集歌谣与故事。
 

characters/base.py

@@ -5,5 +5,5 @@ from dataclasses import dataclass
 class Character:
     name: str
     image_path: str
-    default_timbre: str
+    default_voice: str
     prompt: str

interface.py

@@ -17,8 +17,8 @@ class GradioInterface:
         self.current_svs_model = (
             f"{self.default_config['language']}-{self.default_config['svs_model']}"
         )
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            self.character_info[self.current_character].default_timbre
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            self.character_info[self.current_character].default_voice
         ]
         self.pipeline = SingingDialoguePipeline(self.default_config)
 
@@ -104,21 +104,21 @@ class GradioInterface:
                                 value=self.current_svs_model,
                             )
                         with gr.Row():
-                            timbre_radio = gr.Radio(
-                                label="Singing Timbre",
+                            voice_radio = gr.Radio(
+                                label="Singing voice",
                                 choices=list(
                                     self.svs_model_map[self.current_svs_model][
-                                        "embeddings"
+                                        "voices"
                                     ].keys()
                                 ),
                                 value=self.character_info[
                                     self.current_character
-                                ].default_timbre,
+                                ].default_voice,
                             )
                 character_radio.change(
                     fn=self.update_character,
                     inputs=character_radio,
-                    outputs=[character_image, timbre_radio],
+                    outputs=[character_image, voice_radio],
                 )
                 asr_radio.change(
                     fn=self.update_asr_model, inputs=asr_radio, outputs=asr_radio
@@ -129,15 +129,15 @@ class GradioInterface:
                 svs_radio.change(
                     fn=self.update_svs_model,
                     inputs=svs_radio,
-                    outputs=[svs_radio, timbre_radio],
+                    outputs=[svs_radio, voice_radio],
                 )
                 melody_radio.change(
                     fn=self.update_melody_source,
                     inputs=melody_radio,
                     outputs=melody_radio,
                 )
-                timbre_radio.change(
-                    fn=self.update_timbre, inputs=timbre_radio, outputs=timbre_radio
+                voice_radio.change(
+                    fn=self.update_voice, inputs=voice_radio, outputs=voice_radio
                 )
                 mic_input.change(
                     fn=self.run_pipeline,
@@ -152,12 +152,12 @@ class GradioInterface:
 
     def update_character(self, character):
         self.current_character = character
-        character_timbre = self.character_info[self.current_character].default_timbre
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            character_timbre
+        character_voice = self.character_info[self.current_character].default_voice
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            character_voice
         ]
         return gr.update(value=self.character_info[character].image_path), gr.update(
-            value=character_timbre
+            value=character_voice
         )
 
     def update_asr_model(self, asr_model):
@@ -170,23 +170,23 @@ class GradioInterface:
 
     def update_svs_model(self, svs_model):
         self.current_svs_model = svs_model
-        character_timbre = self.character_info[self.current_character].default_timbre
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            character_timbre
+        character_voice = self.character_info[self.current_character].default_voice
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            character_voice
         ]
         self.pipeline.set_svs_model(
             self.svs_model_map[self.current_svs_model]["model_path"]
         )
         print(
-            f"SVS model updated to {self.current_svs_model}. Will set gradio svs_radio to {svs_model} and timbre_radio to {character_timbre}"
+            f"SVS model updated to {self.current_svs_model}. Will set gradio svs_radio to {svs_model} and voice_radio to {character_voice}"
         )
         return (
             gr.update(value=svs_model),
             gr.update(
                 choices=list(
-                    self.svs_model_map[self.current_svs_model]["embeddings"].keys()
+                    self.svs_model_map[self.current_svs_model]["voices"].keys()
                 ),
-                value=character_timbre,
+                value=character_voice,
             ),
         )
 
@@ -194,20 +194,18 @@ class GradioInterface:
         self.current_melody_source = melody_source
         return gr.update(value=self.current_melody_source)
 
-    def update_timbre(self, timbre):
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            timbre
+    def update_voice(self, voice):
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            voice
         ]
-        return gr.update(value=timbre)
+        return gr.update(value=voice)
 
     def run_pipeline(self, audio_path):
         results = self.pipeline.run(
             audio_path,
             self.svs_model_map[self.current_svs_model]["lang"],
             self.character_info[self.current_character].prompt,
-            svs_inference_kwargs={
-                "speaker": self.current_timbre,
-            },
+            self.current_voice,
             max_new_tokens=100,
         )
         formatted_logs = f"ASR: {results['asr_text']}\nLLM: {results['llm_text']}"

modules/svs/base.py

@@ -13,6 +13,8 @@ class AbstractSVSModel(ABC):
     def synthesize(
         self,
         score: list[tuple[float, float, str, int]],
+        language: str,
+        speaker: str,
         **kwargs,
     ) -> tuple[np.ndarray, int]:
         """

modules/svs/espnet.py

@@ -99,11 +99,11 @@ class ESPNetSVS(AbstractSVSModel):
         return batch
 
     def synthesize(
-        self, score: list[tuple[float, float, str, int]], language: str, **kwargs
+        self, score: list[tuple[float, float, str, int]], language: str, speaker: str, **kwargs
     ):
         batch = self._preprocess(score, language)
         if self.model_id == "espnet/aceopencpop_svs_visinger2_40singer_pretrain":
-            sid = np.array([int(kwargs["speaker"])])
+            sid = np.array([int(speaker)])
             output_dict = self.model(batch, sids=sid)
         elif self.model_id == "espnet/mixdata_svs_visinger2_spkemb_lang_pretrained":
             langs = {
@@ -115,7 +115,7 @@ class ESPNetSVS(AbstractSVSModel):
                     f"Unsupported language: {language} for {self.model_id}"
                 )
             lid = np.array([langs[language]])
-            spk_embed = np.load(kwargs["speaker"])
+            spk_embed = np.load(speaker)
             output_dict = self.model(batch, lids=lid, spembs=spk_embed)
         else:
             raise NotImplementedError(f"Model {self.model_id} not supported")

pipeline.py

@@ -55,7 +55,7 @@ class SingingDialoguePipeline:
         audio_path,
         language,
         prompt_template,
-        svs_inference_kwargs,
+        speaker,
         max_new_tokens=100,
     ):
         if self.track_latency:
@@ -81,7 +81,7 @@ class SingingDialoguePipeline:
         if self.track_latency:
             svs_start_time = time.time()
         singing_audio, sample_rate = self.svs.synthesize(
-            score, language=language, **svs_inference_kwargs
+            score, language=language, speaker=speaker
         )
         if self.track_latency:
             svs_end_time = time.time()