Update app.py

app.py

@@ -13,15 +13,12 @@ from huggingface_hub import HfApi, hf_hub_download, snapshot_download
 from TTS.tts.configs.xtts_config import XttsConfig
 from TTS.tts.models.xtts import Xtts
 from vinorm import TTSnorm
-from langchain_community.llms import HuggingFacePipeline
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
-from gradio_client import Client
-import cv2
-from moviepy.editor import AudioFileClip, ImageSequenceClip
-import gc
-from content_generation import create_content  # Nhập hàm create_content từ file content_generation.py
+from langchain_community.llms import HuggingFacePipeline
 
-# download for mecab
+# Download for mecab
 os.system("python -m unidic download")
 HF_TOKEN = os.environ.get("HF_TOKEN")
 api = HfApi(token=HF_TOKEN)
@@ -58,19 +55,29 @@ supported_languages = config.languages
 if not "vi" in supported_languages:
     supported_languages.append("vi")
 
-# Load LangChain components với mô hình mới
+# Load LangChain components with the new model
 model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-xl")
 tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xl")
 pipe = pipeline(
     'text2text-generation',
     model=model,
     tokenizer=tokenizer,
-    max_length=1024  # Cập nhật max_length
+    max_length=1024  # Update max_length
 )
 local_llm = HuggingFacePipeline(pipeline=pipe)
-llm_chain = caption_chain.chain(llm=local_llm)
-sum_llm_chain = tag_chain.chain(llm=local_llm)
-pexels_api_key = os.getenv('pexels_api_key')
+
+# Define the caption_chain function
+def caption_chain(llm):
+    sum_template = """What is the most significant action, place, or thing? Say it in at most 5 words:
+    
+    {sentence}
+    """
+    sum_prompt = PromptTemplate(template=sum_template, input_variables=["sentence"])
+    sum_llm_chain = LLMChain(prompt=sum_prompt, llm=llm)
+    return sum_llm_chain
+
+# Initialize the caption_chain and tag_chain
+llm_chain = caption_chain(llm=local_llm)
 
 def normalize_vietnamese_text(text):
     text = (
@@ -100,81 +107,23 @@ def calculate_keep_len(text, lang):
         return 13000 * word_count + 2000 * num_punct
     return -1
 
-def create_video_from_audio_and_images(audio_path, images, output_path):
-    audio_clip = AudioFileClip(audio_path)
-    duration = audio_clip.duration
-    
-    # Calculate frame rate based on number of images and audio duration
-    frame_rate = len(images) / duration
-    
-    # Create video clip from images
-    video_clip = ImageSequenceClip(images, fps=frame_rate)
-    
-    # Set audio for video clip
-    final_clip = video_clip.set_audio(audio_clip)
-    
-    # Write result to file
-    final_clip.write_videofile(output_path, codec='libx264', audio_codec='aac')
-    audio_clip.close()
-    video_clip.close()
-    final_clip.close()
-
-def truncate_prompt(prompt, tokenizer, max_length=512):
-    """Truncate prompt to fit within the maximum token length."""
-    tokens = tokenizer.tokenize(prompt)
-    if len(tokens) > max_length:
-        tokens = tokens[:max_length]
-        prompt = tokenizer.convert_tokens_to_string(tokens)
-    return prompt
-
-def generate_images_from_sentences(sentences):
-    try:
-        client = Client("ByteDance/Hyper-FLUX-8Steps-LoRA")
-        for i, sentence in enumerate(sentences):
-            print(f"Generating image for sentence {i + 1}: {sentence}")
-            result = client.predict(
-                height=1024,
-                width=1024,
-                steps=8,
-                scales=3.5,
-                prompt=sentence,
-                seed=3413,
-                api_name="/process_image"
-            )
-            image_path = os.path.join(folder_path, f"image_{i + 1}.png")
-            result.save(image_path)
-            print(f"Saved image at {image_path}")
-    except Exception as e:
-        print("Error! Failed generating images")
-        print(e)
-        return []
-
 @spaces.GPU
 def predict(
     prompt,
     language,
     audio_file_pth,
     normalize_text=True,
-    use_llm=False,  # Thêm tùy chọn sử dụng LLM
-    content_type="Theo yêu cầu",  # Loại nội dung (ví dụ: "triết lý sống" hoặc "Theo yêu cầu")
 ):
-    if use_llm:
-        # Nếu sử dụng LLM, tạo nội dung văn bản từ đầu vào
-        print("I: Generating text with LLM...")
-        generated_text = create_content(prompt, content_type, language)
-        print(f"Generated text: {generated_text}")
-        prompt = generated_text  # Gán văn bản được tạo bởi LLM vào biến prompt
-
     if language not in supported_languages:
         metrics_text = gr.Warning(
             f"Language you put {language} in is not in our Supported Languages, please choose from dropdown"
         )
-        return (None, None, metrics_text)
+        return (None, metrics_text)
 
     speaker_wav = audio_file_pth
     if len(prompt) < 2:
         metrics_text = gr.Warning("Please give a longer prompt text")
-        return (None, None, metrics_text)
+        return (None, metrics_text)
 
     try:
         metrics_text = ""
@@ -194,15 +143,12 @@ def predict(
             metrics_text = gr.Warning(
                 "It appears something wrong with reference, did you unmute your microphone?"
             )
-            return (None, None, metrics_text)
+            return (None, metrics_text)
 
         prompt = re.sub("([^\x00-\x7F]|\w)(\.|\。|\?)", r"\1 \2\2", prompt)
         if normalize_text and language == "vi":
             prompt = normalize_vietnamese_text(prompt)
-        
-        # Truncate prompt to fit within the maximum token length
-        prompt = truncate_prompt(prompt, tokenizer, max_length=512)
-        
+
         print("I: Generating new audio...")
         t0 = time.time()
         out = MODEL.inference(
@@ -227,38 +173,15 @@ def predict(
         keep_len = calculate_keep_len(prompt, language)
         out["wav"] = out["wav"][:keep_len]
         torchaudio.save("output.wav", torch.tensor(out["wav"]).unsqueeze(0), 24000)
-        
-        # Tạo video từ file audio và các cảnh
-        print("I: Generating images from sentences...")
-        # Sử dụng UUID để tạo tên thư mục ngắn gọn
-        folder_name = f"video_{uuid.uuid4().hex}"
-        os.makedirs(folder_name, exist_ok=True)
-        folder_path = os.path.join(folder_name, "images")
-        os.makedirs(folder_path, exist_ok=True)
-        
-        # Tách các câu từ văn bản
-        sentences = [x.strip() for x in re.split(r'[.!?]', prompt) if len(x.strip()) > 6]
-        
-        # Tạo ảnh minh họa cho từng câu
-        images = generate_images_from_sentences(sentences)
-        
-        # Tạo video từ file audio và các ảnh
-        video_path = os.path.join(folder_name, "Final_Ad_Video.mp4")
-        create_video_from_audio_and_images("output.wav", images, video_path)
-        
-        print(f"I: Video generated at {video_path}")
-        metrics_text += f"Video generated at {video_path}\n"
-        
-        return ("output.wav", video_path, metrics_text)
     except RuntimeError as e:
         if "device-side assert" in str(e):
-            # cannot do anything on cuda device side error, need to restart
+            # Cannot do anything on CUDA device side error, need to restart
             print(
                 f"Exit due to: Unrecoverable exception caused by language:{language} prompt:{prompt}",
                 flush=True,
             )
             gr.Warning("Unhandled Exception encounter, please retry in a minute")
-            print("Cuda device-assert Runtime encountered need restart")
+            print("CUDA device-assert Runtime encountered need restart")
             error_time = datetime.datetime.now().strftime("%d-%m-%Y-%H:%M:%S")
             error_data = [
                 error_time,
@@ -273,7 +196,7 @@ def predict(
             csv.writer(write_io).writerows([error_data])
             csv_upload = write_io.getvalue().encode()
             filename = error_time + "_" + str(uuid.uuid4()) + ".csv"
-            print("Writing error csv")
+            print("Writing error CSV")
             error_api = HfApi()
             error_api.upload_file(
                 path_or_fileobj=csv_upload,
@@ -281,7 +204,7 @@ def predict(
                 repo_id="coqui/xtts-flagged-dataset",
                 repo_type="dataset",
             )
-            # speaker_wav
+            # Speaker WAV
             print("Writing error reference audio")
             speaker_filename = error_time + "_reference_" + str(uuid.uuid4()) + ".wav"
             error_api = HfApi()
@@ -308,16 +231,9 @@ def predict(
                 metrics_text = gr.Warning(
                     "Something unexpected happened please retry again."
                 )
-            return (None, None, metrics_text)
-    except Exception as e:
-        print("Unexpected error:", str(e))
-        metrics_text = gr.Warning(
-            "An unexpected error occurred. Please try again later."
-        )
-        return (None, None, metrics_text)
-    return ("output.wav", None, metrics_text)
+            return (None, metrics_text)
+    return ("output.wav", metrics_text)
 
-# Cập nhật giao diện Gradio
 with gr.Blocks(analytics_enabled=False) as demo:
     with gr.Row():
         with gr.Column():
@@ -327,7 +243,7 @@ with gr.Blocks(analytics_enabled=False) as demo:
                 """
             )
         with gr.Column():
-            # placeholder to align the image
+            # Placeholder to align the image
             pass
 
     with gr.Row():
@@ -367,16 +283,6 @@ with gr.Blocks(analytics_enabled=False) as demo:
                 info="Normalize Vietnamese text",
                 value=True,
             )
-            use_llm_checkbox = gr.Checkbox(
-                label="Sử dụng LLM để tạo nội dung",
-                info="Use LLM to generate content",
-                value=False,
-            )
-            content_type_dropdown = gr.Dropdown(
-                label="Loại nội dung",
-                choices=["triết lý sống", "Theo y��u cầu"],
-                value="Theo yêu cầu",
-            )
             ref_gr = gr.Audio(
                 label="Reference Audio (Giọng mẫu)",
                 type="filepath",
@@ -391,7 +297,6 @@ with gr.Blocks(analytics_enabled=False) as demo:
 
         with gr.Column():
             audio_gr = gr.Audio(label="Synthesised Audio", autoplay=True)
-            video_gr = gr.Video(label="Generated Video")
             out_text_gr = gr.Text(label="Metrics")
 
     tts_button.click(
@@ -401,10 +306,8 @@ with gr.Blocks(analytics_enabled=False) as demo:
             language_gr,
             ref_gr,
             normalize_text,
-            use_llm_checkbox,  # Thêm checkbox để bật/tắt LLM
-            content_type_dropdown,  # Thêm dropdown để chọn loại nội dung
         ],
-        outputs=[audio_gr, video_gr, out_text_gr],
+        outputs=[audio_gr, out_text_gr],
         api_name="predict",
     )