Create app.py

app.py

@@ -0,0 +1,219 @@
+import os
+import tempfile
+import requests
+import gradio as gr
+import torch
+import librosa
+import numpy as np
+import subprocess
+import sys
+
+def install_dependencies():
+    """Install required packages"""
+    try:
+        # Clone ss-vq-vae repository
+        if not os.path.exists('ss-vq-vae'):
+            print("Cloning ss-vq-vae repository...")
+            subprocess.run(['git', 'clone', 'https://github.com/cifkao/ss-vq-vae.git'], check=True)
+        
+        # Install ss-vq-vae
+        subprocess.run([sys.executable, '-m', 'pip', 'install', './ss-vq-vae/src'], check=True)
+        
+        print("Dependencies installed successfully!")
+        
+    except Exception as e:
+        print(f"Error installing dependencies: {e}")
+        raise
+
+# Install dependencies
+install_dependencies()
+
+# Import after installation
+import confugue
+from ss_vq_vae.models.vqvae_oneshot import Experiment
+
+def download_model():
+    """Download model files if they don't exist"""
+    model_dir = 'ss-vq-vae/experiments/model'
+    os.makedirs(model_dir, exist_ok=True)
+    
+    model_path = os.path.join(model_dir, 'model_state.pt')
+    if not os.path.exists(model_path):
+        print("Downloading model...")
+        url = 'https://adasp.telecom-paris.fr/rc-ext/demos_companion-pages/vqvae_examples/ssvqvae_model_state.pt'
+        response = requests.get(url)
+        with open(model_path, 'wb') as f:
+            f.write(response.content)
+        print("Model downloaded successfully!")
+
+# Download and initialize model
+download_model()
+logdir = 'ss-vq-vae/experiments/model'
+cfg = confugue.Configuration.from_yaml_file(os.path.join(logdir, 'config.yaml'))
+exp = cfg.configure(Experiment, logdir=logdir, device='cpu')
+exp.model.load_state_dict(torch.load(os.path.join(logdir, 'model_state.pt'), map_location=exp.device))
+exp.model.train(False)
+
+# Preset audio URLs
+INPUT_ROOT = 'https://adasp.telecom-paris.fr/rc-ext/demos_companion-pages/vqvae_examples/'
+INPUT_URLS = {
+    'Electric Guitar': INPUT_ROOT + 'real/content/UnicornRodeo_Maybe_UnicornRodeo_Maybe_Full_25_ElecGtr2CloseMic3.0148.mp3',
+    'Electric Organ': INPUT_ROOT + 'real/style/AllenStone_Naturally_Allen%20Stone_Naturally_Keys-Organ-Active%20DI.0253.mp3',
+    'Jazz Piano': INPUT_ROOT + 'real/style/MaurizioPagnuttiSextet_AllTheGinIsGone_MaurizioPagnuttiSextet_AllTheGinIsGone_Full_12_PianoMics1.08.mp3',
+    'Synth': INPUT_ROOT + 'real/content/Skelpolu_TogetherAlone_Skelpolu_TogetherAlone_Full_13_Synth.0190.mp3',
+    'Rhodes DI': INPUT_ROOT + 'real/content/Diesel13_ColourMeRed_Diesel13_ColourMeRed_Full_30_RhodesDI.0062.mp3',
+    'Acoustic Guitar Lead': INPUT_ROOT + 'real/style/NikolaStajicFtVlasisKostas_Nalim_Nikola%20Stajic%20ft.%20Vlasis%20Kostas_Nalim_Acoustic%20Guitar-Lead-Ela%20M%20251.0170.mp3',
+    'Bass Amp': INPUT_ROOT + 'real/content/HurrayForTheRiffRaff_LivingInTheCity_Hurray%20for%20the%20Riff%20Raff_Livin%20in%20the%20City_Bass-Amp-M82.0018.mp3',
+    'Bass Bip': INPUT_ROOT + 'real/style/RememberDecember_CUNextTime_RememberDecember_CUNextTime_Full_11_Bass_bip.041.mp3',
+    'SynthFX': INPUT_ROOT + 'real/content/MR0902_JamesElder_MR0902_JamesElder_Full_13_SynthFX1.163.mp3',
+    'Electric Guitar Close': INPUT_ROOT + 'real/style/Fergessen_TheWind_Fergessen_TheWind_Full_17_SlecGtr3a_Close.146.mp3',
+    'Rhodes NBATG': INPUT_ROOT + 'real/content/NickiBluhmAndTheGramblers_GoGoGo_NBATG%20-%20Rhodes%20-%20DI.098.mp3',
+    'Keys DI Grace': INPUT_ROOT + 'real/style/JessicaChildress_SlowDown_SD%20KEYS-DI-GRACE.147.mp3',
+    'Dulcimer': INPUT_ROOT + 'real/content/ButterflyEffect_PreachRightHere_ButterflyEffect_PreachRightHere_Full_16_Dulcimer2.076.mp3',
+    'Strings Section': INPUT_ROOT + 'real/style/AngeloBoltini_ThisTown_AngeloBoltini_ThisTown_Full_47_Strings_SectionMic_Vln2.0181.mp3',
+    'Mellotron': INPUT_ROOT + 'real/content/Triviul_Dorothy_Triviul_Dorothy_Full_07_Mellotron.120.mp3',
+    'Acoustic Guitar CU': INPUT_ROOT + 'real/style/UncleDad_WhoIAm_legend-strings_AC%20GUITAR-3-CU29-SHADOWHILL.R.0106.mp3',
+    'Fiddle': INPUT_ROOT + 'real/content/EndaReilly_CurAnLongAgSeol_EndaReilly_CurAnLongAgSeol_Full_10_Fiddle2.0163.mp3',
+    'Violins': INPUT_ROOT + 'real/style/ScottElliott_AeternumVale_ScottElliott_AeternumVale_Full_41_Violins.0138.mp3',
+    'Upright Bass': INPUT_ROOT + 'real/content/AbletonesBigBand_SongOfIndia_UPRIGHT%20BASS%20-%20ELA%20M%20260%20-%20Neve%2033102.136.mp3',
+    'Taiko': INPUT_ROOT + 'real/style/CarlosGonzalez_APlaceForUs_CarlosGonzalez_APlaceForUs_Full_21_Taiko.0115.mp3',
+    'Guitar 2': INPUT_ROOT + 'real/content/AllHandsLost_Ambitions_AllHandsLost_Ambitions_Full_Guitar%202.0292.mp3',
+    'Alto Sax': INPUT_ROOT + 'real/style/SunshineGarciaBand_ForIAmTheMoon_zip5-outro-uke-shaker_OUTRO%20ALTO-251E-SSL6000E.0290.mp3',
+    'Bass Close Mic': INPUT_ROOT + 'real/content/DonCamilloChoir_MarshMarigoldsSong_DonCamilloChoir_MarshMarigoldsSong_Full_08_BassCloseMic2.000.mp3',
+    'Electric Guitar Distorted': INPUT_ROOT + 'real/style/EnterTheHaggis_TwoBareHands_25.%20Jubilee%20Riots%20-%202%20Bar%20Hands_ELE%20Guitars-Ignater-M81.160.mp3',
+    'Bells': INPUT_ROOT + 'real/content/cryonicPAX_Melancholy_cryonicPAX_Melancholy_Full_10_Bells.0034.mp3',
+    'Bass Mic 647': INPUT_ROOT + 'real/style/KungFu_JoyRide_40.%20Kung%20Fu%20-%20Joy%20ride_Bass-Mic-647.0090.mp3',
+}
+
+# Separate content and style options based on URL paths
+CONTENT_OPTIONS = [key for key in INPUT_URLS.keys() if any(word in INPUT_URLS[key] for word in ['content'])]
+STYLE_OPTIONS = [key for key in INPUT_URLS.keys() if any(word in INPUT_URLS[key] for word in ['style'])]
+
+# Add remaining items to both lists if they don't contain 'content' or 'style'
+for key in INPUT_URLS.keys():
+    if key not in CONTENT_OPTIONS and key not in STYLE_OPTIONS:
+        CONTENT_OPTIONS.append(key)
+        STYLE_OPTIONS.append(key)
+
+def load_audio_from_url(url, sr=None):
+    """Load audio from URL by downloading to temporary file"""
+    response = requests.get(url)
+    with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as tmp_file:
+        tmp_file.write(response.content)
+        tmp_file_path = tmp_file.name
+
+    audio, _ = librosa.load(tmp_file_path, sr=sr)
+    os.unlink(tmp_file_path)
+    return audio
+
+def process_timbre_transfer(content_file, content_preset, style_file, style_preset, max_duration=8):
+    """Process timbre transfer with uploaded files or presets"""
+    try:
+        # Load content audio (musical notes/melody to preserve)
+        if content_file is not None:
+            a_content, _ = librosa.load(content_file, sr=exp.sr)
+        else:
+            if content_preset and content_preset in INPUT_URLS:
+                a_content = load_audio_from_url(INPUT_URLS[content_preset], sr=exp.sr)
+            else:
+                return None, "Please upload a content file or select a content preset"
+
+        # Load style audio (timbre/texture to apply)
+        if style_file is not None:
+            a_style, _ = librosa.load(style_file, sr=exp.sr)
+        else:
+            if style_preset and style_preset in INPUT_URLS:
+                a_style = load_audio_from_url(INPUT_URLS[style_preset], sr=exp.sr)
+            else:
+                return None, "Please upload a style file or select a style preset"
+
+        # Limit duration to prevent memory issues
+        max_samples = int(max_duration * exp.sr)
+        if len(a_content) > max_samples:
+            a_content = a_content[:max_samples]
+        if len(a_style) > max_samples:
+            a_style = a_style[:max_samples]
+
+        # Preprocess: Convert audio to model input format
+        s_content = torch.as_tensor(exp.preprocess(a_content), device=exp.device)[None, :]
+        s_style = torch.as_tensor(exp.preprocess(a_style), device=exp.device)[None, :]
+        l_content, l_style = (torch.as_tensor([x.shape[2]], device=exp.device) for x in [s_content, s_style])
+
+        # Run model: Extract content features, extract style features, then recombine
+        with torch.no_grad():
+            s_output = exp.model(input_c=s_content, input_s=s_style,
+                               length_c=l_content, length_s=l_style)
+
+        # Postprocess: Convert model output back to audio waveform
+        a_output = exp.postprocess(s_output.cpu().numpy()[0])
+
+        return (exp.sr, a_output), "Transfer completed successfully!"
+
+    except Exception as e:
+        return None, f"Error: {str(e)}"
+
+# Create Gradio interface
+with gr.Blocks(title="VQ-VAE Timbre Transfer", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("""
+    # 🎵 VQ-VAE Timbre Transfer Demo
+
+    Transfer the timbre (tone/texture) from one audio source to another while preserving the musical content.
+
+    **Content**: Musical notes/melody that will be preserved  
+    **Style**: Instrument timbre/texture that will be applied
+
+    ### How It Works:
+    The model separates musical content from timbre, then recombines them. Some instrument combinations work better than others due to harmonic compatibility and spectral characteristics.
+
+    ### Tips:
+    - Harmonic instruments (piano, guitar) often transfer well to each other
+    - Percussive to sustained transfers may sound less natural
+    - Try different combinations - unexpected results can be musically interesting!
+    """)
+
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("### 🎼 Content Audio")
+            content_file = gr.Audio(label="Upload Content Audio", type="filepath")
+            content_preset = gr.Dropdown(
+                choices=[""] + CONTENT_OPTIONS,
+                label="Or choose preset",
+                value=""
+            )
+
+        with gr.Column():
+            gr.Markdown("### 🎨 Style Audio")
+            style_file = gr.Audio(label="Upload Style Audio", type="filepath")
+            style_preset = gr.Dropdown(
+                choices=[""] + STYLE_OPTIONS,
+                label="Or choose preset",
+                value="Electric Guitar Close"
+            )
+
+    max_duration = gr.Slider(1, 15, value=8, step=1, label="Max Duration (seconds)")
+
+    process_btn = gr.Button("🚀 Transfer Timbre", variant="primary", size="lg")
+
+    with gr.Row():
+        output_audio = gr.Audio(label="🎵 Output Audio", interactive=False)
+        status_msg = gr.Textbox(label="Status", interactive=False, max_lines=3)
+
+    process_btn.click(
+        fn=process_timbre_transfer,
+        inputs=[content_file, content_preset, style_file, style_preset, max_duration],
+        outputs=[output_audio, status_msg]
+    )
+
+    gr.Markdown("""
+    ### 🔧 Troubleshooting
+    - **Poor transfer quality?** Try different instrument combinations or adjust max duration
+    - **Audio doesn't load?** Check internet connection or try different presets
+    - **Processing slow?** Reduce max duration or try shorter audio clips
+    
+    ### 📖 Citation
+    Original work by Ondřej Cífka (InterDigital R&D and Télécom Paris, 2020).  
+    Demo by Ali Dulaimi.
+    """)
+
+if __name__ == "__main__":
+    demo.launch()