Update app.py

app.py

@@ -167,8 +167,7 @@ def generate_filename(prompt, response, file_type="md"):
     Generate filename with meaningful terms and short dense clips from prompt & response.
     The filename should be about 150 chars total, include high-info terms, and a clipped snippet.
     """
-    timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
-    prefix = f"{timestamp}_"
+    prefix = datetime.now().strftime("%y%m_%H%M") + "_"
     combined = (prompt + " " + response).strip()
     info_terms = get_high_info_terms(combined, top_n=10)
     
@@ -194,24 +193,6 @@ def create_file(prompt, response, file_type="md"):
         f.write(prompt + "\n\n" + response)
     return filename
 
-def append_to_transcript(entry: dict):
-    """Append a new entry at the top of transcript.md"""
-    transcript_file = "transcript.md"
-    new_entry = f"1. **Input:** {entry['input']}\n\n   **Output:** {entry['output']}\n\n   **Files:**\n"
-    for file in entry['files']:
-        emoji = FILE_EMOJIS.get(file.split('.')[-1], '')
-        new_entry += f"   - {emoji} [{os.path.basename(file)}]({file})\n"
-    new_entry += "\n---\n\n"
-
-    if os.path.exists(transcript_file):
-        with open(transcript_file, 'r', encoding='utf-8') as f:
-            existing_content = f.read()
-    else:
-        existing_content = ""
-    
-    with open(transcript_file, 'w', encoding='utf-8') as f:
-        f.write(new_entry + existing_content)
-
 def get_download_link(file, file_type="zip"):
     """Generate download link for file"""
     with open(file, "rb") as f:
@@ -231,7 +212,6 @@ def get_download_link(file, file_type="zip"):
 def clean_for_speech(text: str) -> str:
     """Clean text for speech synthesis"""
     text = text.replace("\n", " ")
-    text = text.replace("\r", " ")
     text = text.replace("</s>", " ")
     text = text.replace("#", "")
     text = re.sub(r"\(https?:\/\/[^\)]+\)", "", text)
@@ -259,7 +239,7 @@ async def edge_tts_generate_audio(text, voice="en-US-AriaNeural", rate=0, pitch=
     rate_str = f"{rate:+d}%"
     pitch_str = f"{pitch:+d}Hz"
     communicate = edge_tts.Communicate(text, voice, rate=rate_str, pitch=pitch_str)
-    out_fn = generate_filename(text, text, file_format=file_format)
+    out_fn = generate_filename(text, text, file_type=file_format)
     await communicate.save(out_fn)
     return out_fn
 
@@ -337,16 +317,9 @@ def process_video_with_gpt(video_path, prompt):
 
 # 🤖 9. AI Model Integration
 
-def save_full_transcript(query, text, files):
+def save_full_transcript(query, text):
     """Save full transcript of Arxiv results as a file."""
-    # Sanitize query by replacing carriage returns and line feeds
-    sanitized_query = query.replace('\r', ' ').replace('\n', ' ').strip()
-    entry = {
-        'input': sanitized_query,
-        'output': text.replace('\r', ' ').replace('\n', ' ').strip(),
-        'files': files
-    }
-    append_to_transcript(entry)
+    create_file(query, text, "md")
 
 def parse_arxiv_refs(ref_text: str):
     """
@@ -415,7 +388,6 @@ def create_paper_audio_files(papers, input_question):
     """
     # Collect all content for combined summary
     combined_titles = []
-    created_files = []
 
     for paper in papers:
         try:
@@ -426,7 +398,6 @@ def create_paper_audio_files(papers, input_question):
             file_format = st.session_state['audio_format']
             full_file = speak_with_edge_tts(full_text, voice=st.session_state['tts_voice'], file_format=file_format)
             paper['full_audio'] = full_file
-            created_files.append(full_file)
 
             # Display the audio immediately after generation
             st.write(f"### {FILE_EMOJIS.get(file_format, '')} {os.path.basename(full_file)}")
@@ -446,9 +417,6 @@ def create_paper_audio_files(papers, input_question):
         st.write(f"### {FILE_EMOJIS.get(file_format, '')} Combined Summary Audio")
         play_and_download_audio(combined_file, file_type=file_format)
         papers.append({'title': 'Combined Summary', 'full_audio': combined_file})
-        created_files.append(combined_file)
-    
-    return created_files
 
 def display_papers(papers):
     """
@@ -492,9 +460,8 @@ def perform_ai_lookup(q, vocal_summary=True, extended_refs=False,
 
     # Parse and process papers
     papers = parse_arxiv_refs(refs)
-    created_files = []
     if papers:
-        created_files = create_paper_audio_files(papers, input_question=q)
+        create_paper_audio_files(papers, input_question=q)
         display_papers(papers)
     else:
         st.warning("No papers found in the response.")
@@ -503,9 +470,7 @@ def perform_ai_lookup(q, vocal_summary=True, extended_refs=False,
     st.write(f"**Total Elapsed:** {elapsed:.2f} s")
 
     # Save full transcript
-    create_md_output = create_file(q, result, "md")
-    created_files.append(create_md_output)
-    save_full_transcript(q, result, created_files)
+    create_file(q, result, "md")
     return result
 
 def process_with_gpt(text):
@@ -523,9 +488,7 @@ def process_with_gpt(text):
         )
         ans = c.choices[0].message.content
         st.write("GPT-4o: " + ans)
-        create_md_output = create_file(text, ans, "md")
-        created_files = [create_md_output]
-        save_full_transcript(text, ans, created_files)
+        create_file(text, ans, "md")
         st.session_state.messages.append({"role":"assistant","content":ans})
     return ans
 
@@ -543,19 +506,16 @@ def process_with_claude(text):
         )
         ans = r.content[0].text
         st.write("Claude-3.5: " + ans)
-        create_md_output = create_file(text, ans, "md")
-        created_files = [create_md_output]
-        save_full_transcript(text, ans, created_files)
+        create_file(text, ans, "md")
         st.session_state.chat_history.append({"user":text,"claude":ans})
     return ans
 
 # 📂 10. File Management
 def create_zip_of_files(md_files, mp3_files, wav_files, input_question):
     """Create zip with intelligent naming based on top 10 common words."""
-    # Exclude 'transcript.md'
-    md_files = [f for f in md_files if os.path.basename(f).lower() != 'transcript.md']
+    # Exclude 'readme.md'
+    md_files = [f for f in md_files if os.path.basename(f).lower() != 'readme.md']
     all_files = md_files + mp3_files + wav_files
-
     if not all_files:
         return None
 
@@ -577,7 +537,7 @@ def create_zip_of_files(md_files, mp3_files, wav_files, input_question):
     combined_content = " ".join(all_content)
     info_terms = get_high_info_terms(combined_content, top_n=10)
     
-    timestamp = datetime.now().strftime("%y%m%d%H%M%S")
+    timestamp = datetime.now().strftime("%y%m_%H%M")
     name_text = '_'.join(term.replace(' ', '-') for term in info_terms[:10])
     zip_name = f"{timestamp}_{name_text}.zip"
     
@@ -587,134 +547,93 @@ def create_zip_of_files(md_files, mp3_files, wav_files, input_question):
     
     return zip_name
 
-def load_transcript():
-    """Load and parse transcript.md"""
-    transcript_file = "transcript.md"
-    if not os.path.exists(transcript_file):
-        return []
-    
-    with open(transcript_file, 'r', encoding='utf-8') as f:
-        content = f.read()
-    
-    entries = content.split('\n---\n\n')
-    parsed_entries = []
-    for entry in entries:
-        if not entry.strip():
-            continue
-        match_input = re.search(r'\*\*Input:\*\* (.+)', entry)
-        match_output = re.search(r'\*\*Output:\*\* (.+)', entry)
-        files = re.findall(r'- (.+) \[(.+)\]\((.+)\)', entry)
-        if match_input and match_output:
-            parsed_entries.append({
-                'input': match_input.group(1),
-                'output': match_output.group(1),
-                'files': [file[2] for file in files]
-            })
-    return parsed_entries
-
-def display_file_manager_sidebar():
+def load_files_for_sidebar():
+    """Load and group files for sidebar display"""
+    md_files = glob.glob("*.md")
+    mp3_files = glob.glob("*.mp3")
+    wav_files = glob.glob("*.wav")  # 🆕 Load WAV files
+
+    md_files = [f for f in md_files if os.path.basename(f).lower() != 'readme.md']
+    all_files = md_files + mp3_files + wav_files
+
+    groups = defaultdict(list)
+    for f in all_files:
+        # Treat underscores as spaces and split into words
+        words = os.path.basename(f).replace('_', ' ').split()
+        # Extract keywords from filename
+        keywords = get_high_info_terms(' '.join(words), top_n=5)
+        group_name = '_'.join(keywords) if keywords else 'Miscellaneous'
+        groups[group_name].append(f)
+
+    # Sort groups based on latest file modification time
+    sorted_groups = sorted(groups.items(), key=lambda x: max(os.path.getmtime(f) for f in x[1]), reverse=True)
+    return sorted_groups
+
+def extract_keywords_from_md(files):
+    """Extract keywords from markdown files"""
+    text = ""
+    for f in files:
+        if f.endswith(".md"):
+            c = open(f,'r',encoding='utf-8').read()
+            text += " " + c
+    return get_high_info_terms(text, top_n=5)
+
+def display_file_manager_sidebar(groups_sorted):
     """Display file manager in sidebar"""
     st.sidebar.title("🎵 Audio & Docs Manager")
 
-    groups_sorted = load_transcript()
-    
+    all_md = []
+    all_mp3 = []
+    all_wav = []  # 🆕 List to hold WAV files
+    for group_name, files in groups_sorted:
+        for f in files:
+            if f.endswith(".md"):
+                all_md.append(f)
+            elif f.endswith(".mp3"):
+                all_mp3.append(f)
+            elif f.endswith(".wav"):
+                all_wav.append(f)  # 🆕 Append WAV files
+
     top_bar = st.sidebar.columns(4)  # 🆕 Adjusted columns to accommodate WAV
     with top_bar[0]:
         if st.button("🗑 DelAllMD"):
-            md_files = glob.glob("*.md")
-            md_files = [f for f in md_files if os.path.basename(f).lower() != 'transcript.md']
-            for f in md_files:
+            for f in all_md:
                 os.remove(f)
             st.session_state.should_rerun = True
     with top_bar[1]:
         if st.button("🗑 DelAllMP3"):
-            mp3_files = glob.glob("*.mp3")
-            for f in mp3_files:
+            for f in all_mp3:
                 os.remove(f)
             st.session_state.should_rerun = True
     with top_bar[2]:
         if st.button("🗑 DelAllWAV"):
-            wav_files = glob.glob("*.wav")  # 🆕 Load WAV files
-            for f in wav_files:
+            for f in all_wav:
                 os.remove(f)
             st.session_state.should_rerun = True
     with top_bar[3]:
         if st.button("⬇️ ZipAll"):
-            md_files = glob.glob("*.md")
-            md_files = [f for f in md_files if os.path.basename(f).lower() != 'transcript.md']
-            mp3_files = glob.glob("*.mp3")
-            wav_files = glob.glob("*.wav")  # 🆕 Load WAV files
-            zip_name = create_zip_of_files(md_files, mp3_files, wav_files, input_question=st.session_state.get('last_query', ''))
+            zip_name = create_zip_of_files(all_md, all_mp3, all_wav, input_question=st.session_state.get('last_query', ''))
             if zip_name:
                 st.sidebar.markdown(get_download_link(zip_name, file_type="zip"), unsafe_allow_html=True)
 
-    for idx, entry in enumerate(groups_sorted, 1):
-        group_name = f"Query {idx}: {entry['input'][:50]}..."  # Truncate for display
-        with st.sidebar.expander(f"{FILE_EMOJIS.get('md', '')} {group_name}", expanded=False):
+    for group_name, files in groups_sorted:
+        keywords_str = group_name.replace('_', ' ') if group_name else "No Keywords"
+        with st.sidebar.expander(f"{FILE_EMOJIS.get('md', '')} {group_name} Files ({len(files)}) - KW: {keywords_str}", expanded=True):
             c1,c2 = st.columns(2)
             with c1:
-                if st.button("👀ViewGrp", key="view_group_"+str(idx)):
-                    st.session_state.viewing_prefix = idx
+                if st.button("👀ViewGrp", key="view_group_"+group_name):
+                    st.session_state.viewing_prefix = group_name
             with c2:
-                if st.button("🗑DelGrp", key="del_group_"+str(idx)):
-                    # Delete associated files
-                    for f in entry['files']:
-                        if os.path.exists(f):
-                            os.remove(f)
-                    # Reload transcript without this entry
-                    all_entries = load_transcript()
-                    del all_entries[idx-1]
-                    # Rewrite transcript.md
-                    transcript_file = "transcript.md"
-                    with open(transcript_file, 'w', encoding='utf-8') as tf:
-                        for i, e in enumerate(all_entries, 1):
-                            new_entry = f"1. **Input:** {e['input']}\n\n   **Output:** {e['output']}\n\n   **Files:**\n"
-                            for file in e['files']:
-                                emoji = FILE_EMOJIS.get(file.split('.')[-1], '')
-                                new_entry += f"   - {emoji} [{os.path.basename(file)}]({file})\n"
-                            new_entry += "\n---\n\n"
-                            tf.write(new_entry)
+                if st.button("🗑DelGrp", key="del_group_"+group_name):
+                    for f in files:
+                        os.remove(f)
                     st.success(f"Deleted group {group_name}!")
                     st.session_state.should_rerun = True
 
-            for f in entry['files']:
+            for f in files:
                 fname = os.path.basename(f)
                 ctime = datetime.fromtimestamp(os.path.getmtime(f)).strftime("%Y-%m-%d %H:%M:%S")
-                st.markdown(f"**{fname}** - {ctime}")
-
-def extract_keywords_from_md(files):
-    """Extract keywords from markdown files"""
-    text = ""
-    for f in files:
-        if f.endswith(".md"):
-            c = open(f,'r',encoding='utf-8').read()
-            text += " " + c
-    return get_high_info_terms(text, top_n=5)
-
-def display_viewing_group(group_idx):
-    """Display the contents of a viewing group"""
-    groups_sorted = load_transcript()
-    if group_idx < 1 or group_idx > len(groups_sorted):
-        st.error("Invalid group selected.")
-        return
-    entry = groups_sorted[group_idx-1]
-    st.write("---")
-    st.write(f"**Viewing Group {group_idx}:** {entry['input']}")
-    for f in entry['files']:
-        fname = os.path.basename(f)
-        ext = os.path.splitext(fname)[1].lower().strip('.')
-        st.write(f"### {fname}")
-        if ext == "md":
-            content = open(f,'r',encoding='utf-8').read()
-            st.markdown(content)
-        elif ext == "mp3":
-            st.audio(f)
-        elif ext == "wav":
-            st.audio(f)  # 🆕 Handle WAV files
-        else:
-            st.markdown(get_download_link(f), unsafe_allow_html=True)
-    if st.button("❌ Close"):
-        st.session_state.viewing_prefix = None
+                st.write(f"**{fname}** - {ctime}")
 
 # 🎯 11. Main Application
 def main():
@@ -807,8 +726,7 @@ def main():
             result = perform_ai_lookup(q, vocal_summary=vocal_summary, extended_refs=extended_refs, 
                                        titles_summary=titles_summary, full_audio=full_audio)
             if full_transcript:
-                # The transcript is already being handled in perform_ai_lookup
-                pass
+                save_full_transcript(q, result)
 
         st.markdown("### Change Prompt & Re-Run")
         q_new = st.text_input("🔄 Modify Query:")
@@ -817,8 +735,7 @@ def main():
             result = perform_ai_lookup(q_new, vocal_summary=vocal_summary, extended_refs=extended_refs, 
                                        titles_summary=titles_summary, full_audio=full_audio)
             if full_transcript:
-                # The transcript is already being handled in perform_ai_lookup
-                pass
+                save_full_transcript(q_new, result)
 
     elif tab_main == "🎤 Voice":
         st.subheader("🎤 Voice Input")
@@ -878,16 +795,34 @@ def main():
             st.write("Select a file from the sidebar to edit.")
 
     # Load and display files in the sidebar
-    display_file_manager_sidebar()
+    groups_sorted = load_files_for_sidebar()
+    display_file_manager_sidebar(groups_sorted)
+
+    if st.session_state.viewing_prefix and any(st.session_state.viewing_prefix == group for group, _ in groups_sorted):
+        st.write("---")
+        st.write(f"**Viewing Group:** {st.session_state.viewing_prefix}")
+        for group_name, files in groups_sorted:
+            if group_name == st.session_state.viewing_prefix:
+                for f in files:
+                    fname = os.path.basename(f)
+                    ext = os.path.splitext(fname)[1].lower().strip('.')
+                    st.write(f"### {fname}")
+                    if ext == "md":
+                        content = open(f,'r',encoding='utf-8').read()
+                        st.markdown(content)
+                    elif ext == "mp3":
+                        st.audio(f)
+                    elif ext == "wav":
+                        st.audio(f)  # 🆕 Handle WAV files
+                    else:
+                        st.markdown(get_download_link(f), unsafe_allow_html=True)
+                break
+        if st.button("❌ Close"):
+            st.session_state.viewing_prefix = None
 
-    # Display viewing group if selected
-    if st.session_state.viewing_prefix:
-        display_viewing_group(st.session_state.viewing_prefix)
-
-    # Additional Markdown in sidebar
     markdownPapers = """
     
-# Levels of AGI
+    # Levels of AGI
 
 ## 1. Performance (rows) x Generality (columns)
 - **Narrow**  
@@ -962,7 +897,7 @@ def main():
     - *Reference:* Stockfish (2023). **Stockfish Chess Engine**. [Website](https://stockfishchess.org)  
 - **Artificial Superintelligence (ASI)**  
   - Not yet achieved  
-    
+
 
 # 🧬 Innovative Architecture of AlphaFold2: A Hybrid System
 
@@ -1009,7 +944,7 @@ def main():
 
     """
     st.sidebar.markdown(markdownPapers)
-
+    
     if st.session_state.should_rerun:
         st.session_state.should_rerun = False
         st.rerun()