Fixed audio sampling issues

app.py

@@ -94,7 +94,9 @@ with gr.Blocks() as demo:
                 with gr.Row():
                     # Add audio
                     audio_upload = gr.Audio(source="upload", type="filepath")
-                    audio_mic = gr.Audio(source="microphone", type="filepath")
+                    audio_mic = gr.Audio(
+                        source="microphone", type="filepath", format="mp3"
+                    )
 
         with gr.Column(scale=8):
             with gr.Box():
@@ -123,4 +125,4 @@ with gr.Blocks() as demo:
         outputs=[prompt, image, audio_upload, audio_mic, chatbot],
     )
 
-demo.launch()
+demo.launch(server_port=8881)

inference/main.py

@@ -1,3 +1,5 @@
+import soundfile as sf
+import librosa
 import torch
 from transformers import (
     AutoTokenizer,
@@ -47,8 +49,14 @@ class WhisperWithProjection:
         # self.audio_language_connector = AudioLanguageConnector(projection_dim)
 
     def __call__(self, audio):
+        array, sampling_rate = sf.read(audio)
+        resampled_array = librosa.resample(
+            array,
+            orig_sr=sampling_rate,
+            target_sr=16000,
+        )
         input_features = self.processor(
-            audio["array"], sampling_rate=audio["sampling_rate"], return_tensors="pt"
+            resampled_array, sampling_rate=16000, return_tensors="pt"
         ).input_features
         # generate token ids
         predicted_ids = self.model.generate(input_features.to(self.device))

requirements.txt

@@ -16,4 +16,6 @@ transformers==4.36.2
 accelerate==0.21.0
 bitsandbytes==0.41.0
 scikit-learn==1.2.2
-sentencepiece==0.1.99
+sentencepiece==0.1.99
+librosa
+soundfile