fix lang detection

audio_processor.py

@@ -10,6 +10,7 @@ from typing import List, Dict, Any
 from timed_objects import ASRToken
 from whisper_streaming_custom.whisper_online import online_factory
 from core import WhisperLiveKit
+import librosa
 
 # Set up logging once
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -406,4 +407,25 @@ class AudioProcessor:
             logger.warning(f"Error writing to FFmpeg: {e}. Restarting...")
             await self.restart_ffmpeg()
             self.ffmpeg_process.stdin.write(message)
-            self.ffmpeg_process.stdin.flush()
+            self.ffmpeg_process.stdin.flush()
+
+    async def detect_language(self, file_path):
+        """Detect the language of the audio file.
+        
+        Args:
+            file_path: Path to the audio file
+            
+        Returns:
+            tuple: (detected_language, confidence, probabilities)
+        """
+        try:
+            # Use the ASR backend to detect language
+            if self.asr:
+                return self.asr.detect_language(file_path)
+            else:
+                raise RuntimeError("ASR backend not initialized")
+                
+        except Exception as e:
+            logger.error(f"Error in language detection: {e}")
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            raise

main.py

@@ -11,8 +11,6 @@ import traceback
 import argparse
 import uvicorn
 import numpy as np
-import librosa
-import io
 import tempfile
 
 from core import WhisperLiveKit
@@ -64,14 +62,8 @@ async def detect_language(file: UploadFile = File(...)):
         
         # Use the audio processor for language detection
         if audio_processor:
-            # Load audio using librosa
-            audio, sr = librosa.load(file_path, sr=16000)
-            
-            # Convert to format expected by Whisper
-            audio = (audio * 32768).astype(np.int16)
-            
-            # Detect language
-            detected_lang, confidence, probs = audio_processor.detect_language(audio)
+            # Detect language using the audio processor
+            detected_lang, confidence, probs = await audio_processor.detect_language(file_path)
             
             # Clean up - remove the temporary file
             os.remove(file_path)

whisper_streaming_custom/backends.py

@@ -89,7 +89,7 @@ class WhisperTimestampedASR(ASRBase):
     def set_translate_task(self):
         self.transcribe_kargs["task"] = "translate"
 
-    def detect_language(self, audio):
+    def detect_language(self, audio_file_path):
         import whisper
         """
         Detect the language of the audio using Whisper's language detection.
@@ -103,12 +103,9 @@ class WhisperTimestampedASR(ASRBase):
                 - confidence (float): Confidence score for the detected language
                 - probabilities (dict): Dictionary of language probabilities
         """
-        try:
-            # Ensure audio is in the correct format
-            if not isinstance(audio, np.ndarray):
-                audio = np.array(audio)
-            
+        try:            
             # Pad or trim audio to the correct length
+            audio = whisper.load_audio(audio_file_path)
             audio = whisper.pad_or_trim(audio)
             
             # Create mel spectrogram with correct dimensions
@@ -183,12 +180,12 @@ class FasterWhisperASR(ASRBase):
     def set_translate_task(self):
         self.transcribe_kargs["task"] = "translate"
 
-    def detect_language(self, audio):
+    def detect_language(self, audio_file_path):
         """
         Detect the language of the audio using faster-whisper's language detection.
         
         Args:
-            audio (np.ndarray): Audio data as numpy array
+            audio_file_path: Path to the audio file
             
         Returns:
             tuple: (detected_language, confidence, probabilities)
@@ -197,9 +194,11 @@ class FasterWhisperASR(ASRBase):
                 - probabilities (dict): Dictionary of language probabilities
         """
         try:
-            # Ensure audio is in the correct format
-            if not isinstance(audio, np.ndarray):
-                audio = np.array(audio)
+            # Load audio using soundfile
+            audio, sr = sf.read(audio_file_path)
+            
+            # Convert to format expected by Whisper (16-bit PCM)
+            audio = (audio * 32768).astype(np.int16)
             
             # Use faster-whisper's detect_language method
             language, language_probability, all_language_probs = self.model.detect_language(