Update src/streamlit_app.py

src/streamlit_app.py

@@ -25,34 +25,68 @@ def load_model():
     return SentenceTransformer('all-MiniLM-L6-v2', cache_folder='/tmp/huggingface')
 
 @st.cache_data(ttl=3600, show_spinner=False)
-def search_openalex_papers(query, num_results=50, country_code=None):
+def search_openalex_papers(query, num_results=50, country_code=None, use_fulltext=False):
     """
     Search OpenAlex for papers related to the query
     Optionally filter by author's country
+    Optionally use full-text search (searches title + abstract + full text when available)
     
-    Note: Results are cached for 1 hour based on query, num_results, and country_code
+    Note: Results are cached for 1 hour based on query, num_results, country_code, and use_fulltext
+    For large requests (>100), uses pagination
     """
     base_url = "https://api.openalex.org/works"
+    all_papers = []
     
-    params = {
-        "search": query,
-        "per_page": num_results,
-        "select": "id,title,abstract_inverted_index,authorships,publication_year,cited_by_count,display_name",
-        "mailto": "[email protected]"  # Polite pool
-    }
+    # OpenAlex max per_page is 200, so we need pagination for large requests
+    per_page = min(200, num_results)
+    num_pages = (num_results + per_page - 1) // per_page  # Ceiling division
     
-    # Add country filter if specified
-    if country_code:
-        params["filter"] = f"authorships.countries:{country_code}"
+    for page in range(1, num_pages + 1):
+        params = {
+            "per_page": per_page,
+            "page": page,
+            "select": "id,title,abstract_inverted_index,authorships,publication_year,cited_by_count,display_name",
+            "mailto": "[email protected]"  # Polite pool
+        }
+        
+        # Build filter string
+        filters = []
+        
+        if use_fulltext:
+            # Full-text search (searches title + abstract + full text when available)
+            filters.append(f"fulltext.search:{query}")
+        else:
+            # Standard search (title + abstract only)
+            params["search"] = query
+        
+        # Add country filter if specified
+        if country_code:
+            filters.append(f"authorships.countries:{country_code}")
+        
+        # Combine filters with comma (AND operation)
+        if filters:
+            params["filter"] = ",".join(filters)
+        
+        try:
+            response = requests.get(base_url, params=params, timeout=30)
+            response.raise_for_status()
+            data = response.json()
+            papers = data.get("results", [])
+            all_papers.extend(papers)
+            
+            # If we got fewer papers than requested, no more pages available
+            if len(papers) < per_page:
+                break
+                
+            # Rate limiting - be nice to OpenAlex
+            if page < num_pages:
+                time.sleep(0.1)  # 100ms delay between requests
+                
+        except Exception as e:
+            st.error(f"Error fetching papers (page {page}): {str(e)}")
+            break
     
-    try:
-        response = requests.get(base_url, params=params, timeout=30)
-        response.raise_for_status()
-        data = response.json()
-        return data.get("results", [])
-    except Exception as e:
-        st.error(f"Error fetching papers: {str(e)}")
-        return []
+    return all_papers[:num_results]  # Return exactly what was requested
 
 def reconstruct_abstract(inverted_index):
     """
@@ -247,14 +281,39 @@ def main():
     
     country_code = country_options[selected_country]
     
-    num_papers = st.sidebar.slider(
-        "Number of papers to fetch",
-        min_value=20,
-        max_value=100,
-        value=50,
-        step=10
+    # Search mode selection
+    search_mode = st.sidebar.radio(
+        "Search Mode",
+        options=["Quick Search", "Deep Search"],
+        help="Quick: 50-100 papers in 30s | Deep: 1,000-5,000 papers in 2-5 min"
     )
     
+    # Full-text search option
+    use_fulltext = st.sidebar.checkbox(
+        "Include full text (when available)",
+        value=False,
+        help="Search title + abstract + full text. Full text available for ~10-15% of papers. May find more specific matches."
+    )
+    
+    if search_mode == "Quick Search":
+        num_papers = st.sidebar.slider(
+            "Number of papers to fetch",
+            min_value=20,
+            max_value=100,
+            value=50,
+            step=10
+        )
+    else:  # Deep Search
+        num_papers = st.sidebar.slider(
+            "Number of papers to fetch",
+            min_value=500,
+            max_value=5000,
+            value=1000,
+            step=500,
+            help="⚠️ Deep search takes 2-5 minutes"
+        )
+        st.sidebar.warning("⚠️ Deep search will take 2-5 minutes to complete")
+    
     top_papers_display = st.sidebar.slider(
         "Top papers to display",
         min_value=5,
@@ -291,16 +350,24 @@ def main():
     
     if search_button and query:
         # Display search parameters
-        st.info(f"🔍 Searching: **{query}** | Papers: **{num_papers}** | Country: **{selected_country}** | Min papers per author: **{min_papers_per_author}**")
+        st.info(f"🔍 Searching: **{query}** | Mode: **{search_mode}** | Papers: **{num_papers}** | Country: **{selected_country}** | Full-text: **{'Yes' if use_fulltext else 'No'}** | Min papers/author: **{min_papers_per_author}**")
         
         # Load model
         with st.spinner("Loading semantic model..."):
             model = load_model()
         
         # Search papers
-        search_key = f"{query}_{num_papers}_{country_code}"
-        with st.spinner(f"Searching OpenAlex for papers about '{query}'{' from ' + selected_country if country_code else ''}..."):
-            papers = search_openalex_papers(query, num_papers, country_code)
+        search_key = f"{query}_{num_papers}_{country_code}_{use_fulltext}"
+        
+        if search_mode == "Deep Search":
+            progress_text = f"🔍 Deep search in progress: Fetching up to {num_papers} papers from OpenAlex..."
+            progress_bar = st.progress(0, text=progress_text)
+        
+        with st.spinner(f"Searching OpenAlex for papers about '{query}'{' from ' + selected_country if country_code else ''}{' (including full text)' if use_fulltext else ''}..."):
+            papers = search_openalex_papers(query, num_papers, country_code, use_fulltext)
+            
+        if search_mode == "Deep Search":
+            progress_bar.progress(33, text="📄 Papers fetched! Now generating embeddings...")
         
         if not papers:
             st.warning("No papers found. Try different search terms.")
@@ -310,14 +377,19 @@ def main():
         
         # Show debug info in expander
         with st.expander("🔍 Search Details", expanded=False):
+            st.write(f"**Search Mode:** {search_mode}")
             st.write(f"**Query:** {query}")
-            st.write(f"**Papers fetched:** {num_papers}")
+            st.write(f"**Full-text search:** {'Enabled' if use_fulltext else 'Disabled'}")
+            st.write(f"**Papers requested:** {num_papers}")
+            st.write(f"**Papers fetched:** {len(papers)}")
             st.write(f"**Country filter:** {selected_country} ({country_code or 'None'})")
-            st.write(f"**Papers returned:** {len(papers)}")
             st.write(f"**First paper:** {papers[0].get('display_name', 'N/A')[:100]}...")
             st.write(f"**Last paper:** {papers[-1].get('display_name', 'N/A')[:100]}...")
         
         # Prepare papers for semantic search
+        if search_mode == "Deep Search":
+            progress_bar.progress(50, text="🧠 Generating semantic embeddings...")
+            
         with st.spinner("Analyzing papers with semantic search..."):
             paper_texts = []
             valid_papers = []
@@ -339,7 +411,11 @@ def main():
             
             # Generate embeddings
             query_embedding = model.encode(query, convert_to_tensor=False)
-            paper_embeddings = model.encode(paper_texts, convert_to_tensor=False, show_progress_bar=True)
+            
+            if search_mode == "Deep Search":
+                progress_bar.progress(66, text=f"🔢 Computing similarity for {len(paper_texts)} papers...")
+            
+            paper_embeddings = model.encode(paper_texts, convert_to_tensor=False, show_progress_bar=False)
             
             # Calculate similarities
             similarities = calculate_semantic_similarity(query_embedding, paper_embeddings)
@@ -348,6 +424,11 @@ def main():
             sorted_indices = np.argsort(similarities)[::-1]
             sorted_papers = [valid_papers[i] for i in sorted_indices]
             sorted_scores = [similarities[i] for i in sorted_indices]
+            
+        if search_mode == "Deep Search":
+            progress_bar.progress(100, text="✅ Complete!")
+            time.sleep(0.5)
+            progress_bar.empty()
         
         # Display top papers
         st.header(f"📄 Top {top_papers_display} Most Relevant Papers")