Create app

app

@@ -0,0 +1,200 @@
+import gradio as gr
+import torch
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+import time
+import os
+import numpy as np
+import soundfile as sf
+import librosa
+
+# --- Configuration ---
+# Device selection (GPU if available, else CPU)
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+print(f"Using device: {device}")
+
+# STT Model (Use smaller model for lower latency)
+stt_model_id = "openai/whisper-tiny" # Or "openai/whisper-base". Avoid larger models for streaming.
+
+# Summarization Model
+summarizer_model_id = "sshleifer/distilbart-cnn-6-6" # Use a distilled/smaller model for speed
+
+# Summarization Interval (seconds) - How often to regenerate the summary
+SUMMARY_INTERVAL = 30.0 # Summarize every 30 seconds
+
+# --- Load Models ---
+# (Keep the model loading code exactly the same as before)
+print("Loading STT model...")
+stt_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+    stt_model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True
+)
+stt_model.to(device)
+processor = AutoProcessor.from_pretrained(stt_model_id)
+stt_pipeline = pipeline(
+    "automatic-speech-recognition",
+    model=stt_model,
+    tokenizer=processor.tokenizer,
+    feature_extractor=processor.feature_extractor,
+    max_new_tokens=128,
+    chunk_length_s=30,
+    batch_size=16,
+    torch_dtype=torch_dtype,
+    device=device,
+)
+print("STT model loaded.")
+
+print("Loading Summarization pipeline...")
+summarizer = pipeline(
+    "summarization",
+    model=summarizer_model_id,
+    device=device
+)
+print("Summarization pipeline loaded.")
+
+
+# --- Helper Functions ---
+# (Keep the format_summary_as_bullets function exactly the same)
+def format_summary_as_bullets(summary_text):
+    """Attempts to format a summary text block into bullet points."""
+    if not summary_text:
+        return ""
+    # Simple approach: split by sentences and add bullets.
+    # More advanced NLP could be used here.
+    sentences = summary_text.replace(". ", ".\n- ").split('\n')
+    bullet_summary = "- " + "\n".join(sentences).strip()
+    # Remove potential empty bullets
+    bullet_summary = "\n".join([line for line in bullet_summary.split('\n') if line.strip() not in ['-', '']])
+    return bullet_summary
+
+
+# --- Processing Function for Streaming ---
+# (Keep the process_audio_stream function exactly the same)
+# This function ONLY processes audio, it doesn't interact with the webcam video
+def process_audio_stream(
+    new_chunk_tuple, # Gradio streaming yields (sample_rate, numpy_data)
+    accumulated_transcript_state, # gr.State holding the full text
+    last_summary_time_state,    # gr.State holding the timestamp of the last summary
+    current_summary_state       # gr.State holding the last generated summary
+    ):
+
+    if new_chunk_tuple is None:
+        # Initial call or stream ended, return current state
+        return accumulated_transcript_state, current_summary_state, accumulated_transcript_state, last_summary_time_state, current_summary_state
+
+    sample_rate, audio_chunk = new_chunk_tuple
+
+    if audio_chunk is None or sample_rate is None or audio_chunk.size == 0:
+        # Handle potential empty chunks gracefully
+        return accumulated_transcript_state, current_summary_state, accumulated_transcript_state, last_summary_time_state, current_summary_state
+
+    print(f"Received chunk: {audio_chunk.shape}, Sample Rate: {sample_rate}, Duration: {len(audio_chunk)/sample_rate:.2f}s")
+
+    # Ensure audio is float32 and mono, as Whisper expects
+    if audio_chunk.dtype != np.float32:
+        # Normalize assuming input is int16
+        # Adjust if your microphone provides different integer types
+        audio_chunk = audio_chunk.astype(np.float32) / 32768.0 # Max value for int16 is 32767
+
+    # --- 1. Transcribe the new chunk ---
+    new_text = ""
+    try:
+        result = stt_pipeline({"sampling_rate": sample_rate, "raw": audio_chunk.copy()})
+        new_text = result["text"].strip() if result["text"] else ""
+        print(f"Transcription chunk: '{new_text}'")
+
+    except Exception as e:
+        print(f"Error during transcription chunk: {e}")
+        new_text = f"[Transcription Error: {e}]"
+
+    # --- 2. Update Accumulated Transcript ---
+    if accumulated_transcript_state and not accumulated_transcript_state.endswith((" ", "\n")) and new_text:
+         updated_transcript = accumulated_transcript_state + " " + new_text
+    else:
+         updated_transcript = accumulated_transcript_state + new_text
+
+    # --- 3. Periodic Summarization ---
+    current_time = time.time()
+    new_summary = current_summary_state # Keep the old summary by default
+    updated_last_summary_time = last_summary_time_state
+
+    # Check transcript length to avoid summarizing tiny bits of text too early
+    if updated_transcript and len(updated_transcript) > 50 and (current_time - last_summary_time_state > SUMMARY_INTERVAL):
+        print(f"Summarizing transcript (length: {len(updated_transcript)})...")
+        try:
+            # Summarize the *entire* transcript up to this point
+            summary_result = summarizer(updated_transcript, max_length=150, min_length=30, do_sample=False)
+            if summary_result and isinstance(summary_result, list):
+                 raw_summary = summary_result[0]['summary_text']
+                 new_summary = format_summary_as_bullets(raw_summary)
+                 updated_last_summary_time = current_time # Update time only on successful summary
+                 print("Summary updated.")
+            else:
+                 print("Summarization did not produce expected output.")
+
+        except Exception as e:
+            print(f"Error during summarization: {e}")
+            # Display error in summary box but keep the last known good summary in state
+            # To avoid overwriting a potentially useful summary with just an error message
+            # We return the error message for display, but not update summary_state with it
+            error_display_summary = f"[Summarization Error]\n\nLast good summary:\n{current_summary_state}"
+            return updated_transcript, error_display_summary, updated_transcript, last_summary_time_state, current_summary_state
+
+
+    # --- 4. Return Updated State and Outputs ---
+    return updated_transcript, new_summary, updated_transcript, updated_last_summary_time, new_summary
+
+
+# --- Gradio Interface ---
+print("Creating Gradio interface...")
+with gr.Blocks() as demo:
+    gr.Markdown("# Real-Time Meeting Notes with Webcam View")
+    gr.Markdown("Speak into your microphone. Transcription appears below. Summary updates periodically.")
+
+    # State variables to store data between stream calls
+    transcript_state = gr.State("") # Holds the full transcript
+    last_summary_time = gr.State(0.0) # Holds the time the summary was last generated
+    summary_state = gr.State("")    # Holds the current bullet point summary
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            # Input: Microphone stream
+            audio_stream = gr.Audio(sources=["microphone"], streaming=True, label="Live Microphone Input", type="numpy")
+
+            # NEW: Webcam Display
+            # Use gr.Image which is simpler for just displaying webcam feed
+            # live=True makes it update continuously
+            webcam_view = gr.Image(sources=["webcam"], label="Your Webcam", streaming=True) # Use streaming=True for live view
+
+        with gr.Column(scale=2):
+             transcription_output = gr.Textbox(label="Full Transcription", lines=15, interactive=False) # Display only
+             summary_output = gr.Textbox(label=f"Bullet Point Summary (Updates ~every {SUMMARY_INTERVAL}s)", lines=10, interactive=False) # Display only
+
+
+    # Connect the streaming audio input to the processing function
+    # Note: The webcam component runs independently in the browser, it doesn't feed data here
+    audio_stream.stream(
+        fn=process_audio_stream,
+        inputs=[audio_stream, transcript_state, last_summary_time, summary_state],
+        outputs=[transcription_output, summary_output, transcript_state, last_summary_time, summary_state],
+    )
+
+    # Add a button to clear the state if needed
+    def clear_state_values():
+        print("Clearing state.")
+        return "", "", 0.0, "" # Clear transcript display, summary display, reset time state, clear summary state
+    # Need separate function to clear states vs displays if they differ
+    def clear_state():
+        return "", 0.0, "" # Clear transcript_state, last_summary_time, summary_state
+
+    clear_button = gr.Button("Clear Transcript & Summary")
+    # This button clears the display textboxes AND resets the internal states
+    clear_button.click(
+        fn=lambda: ("", "", "", 0.0, ""), # Return empty values for all outputs/states
+        inputs=[],
+        outputs=[transcription_output, summary_output, transcript_state, last_summary_time, summary_state]
+    )
+
+
+print("Launching Gradio interface...")
+demo.queue() # Enable queue for handling multiple requests/stream chunks
+demo.launch(debug=True, share=True) # share=True for Colab public link