Update app.py

app.py

@@ -3,18 +3,19 @@ import matplotlib.pyplot as plt
 import nltk
 from nltk.tokenize import word_tokenize, sent_tokenize
 from nltk.corpus import stopwords
-from nltk.stem import WordNetLemmatizer
+# from nltk.stem import WordNetLemmatizer # Not used, commented out
 from nltk.text import Text
 from nltk.probability import FreqDist
 from cleantext import clean
-import textract
+# import textract # Replaced by PyPDF2
+import PyPDF2 # Added for PDF parsing
 import urllib.request
 from io import BytesIO
 import sys
 import pandas as pd
-import cv2
+# import cv2 # Not used, commented out
 import re
-from wordcloud import WordCloud, ImageColorGenerator
+from wordcloud import WordCloud # , ImageColorGenerator # ImageColorGenerator not used, commented out
 from textblob import TextBlob
 from PIL import Image
 import os
@@ -28,37 +29,14 @@ import unidecode
 import contractions
 from sklearn.feature_extraction.text import TfidfVectorizer
 
-
+# Load environment variables
 load_dotenv()
-import nltk
-import ssl
-
-def ensure_nltk_resources():
-    try:
-        nltk.data.find('tokenizers/punkt')
-        nltk.data.find('corpora/stopwords')
-    except LookupError:
-        print("NLTK resources not found. Downloading...")
-        try:
-            # Handling potential SSL issues (common on some systems)
-            _create_unverified_https_context = ssl._create_unverified_context
-        except AttributeError:
-            pass
-        else:
-            ssl._create_default_https_context = _create_unverified_https_context
-
-        nltk.download(['stopwords', 'wordnet', 'words'])
-        nltk.download('punkt')
-        nltk.download('punkt_tab')
-        print("NLTK resources downloaded successfully.")
-
-
-ensure_nltk_resources()
 
 # Download NLTK resources (Ensure this runs once or handle caching)
 # nltk.download(['stopwords', 'wordnet', 'words'])
 # nltk.download('punkt')
 # nltk.download('punkt_tab')
+
 # Initialize Groq client
 groq_api_key = os.getenv("GROQ_API_KEY")
 groq_client = groq.Groq(api_key=groq_api_key) if groq_api_key else None
@@ -68,16 +46,36 @@ stop_words = set(stopwords.words('english'))
 stop_words.update({'ask', 'much', 'thank', 'etc.', 'e', 'We', 'In', 'ed', 'pa', 'This', 'also', 'A', 'fu', 'To', '5', 'ing', 'er', '2'}) # Ensure stop_words is a set
 
 # --- Parsing & Preprocessing Functions ---
+# --- Replaced textract with PyPDF2 ---
 def Parsing(parsed_text):
+    """
+    Parses text from a PDF file using PyPDF2.
+    """
     try:
+        # Get the file path from the Gradio UploadFile object
         if hasattr(parsed_text, 'name'):
             file_path = parsed_text.name
         else:
+            # Fallback if it's somehow just a string path
             file_path = parsed_text
-        # Ensure textract handles encoding correctly or handle errors
-        raw_party = textract.process(file_path) # Removed encoding/method for broader compatibility
-        decoded_text = raw_party.decode('utf-8', errors='ignore') # Decode bytes to string, handling errors
-        return clean(decoded_text) # Pass decoded string to clean
+
+        # Use PyPDF2 to read the PDF
+        text = ""
+        with open(file_path, 'rb') as pdf_file: # Open in binary read mode
+            pdf_reader = PyPDF2.PdfReader(pdf_file)
+            for page_num in range(len(pdf_reader.pages)):
+                page = pdf_reader.pages[page_num]
+                text += page.extract_text() + "\n" # Add newline between pages
+
+        # Clean the extracted text
+        return clean(text)
+
+    except FileNotFoundError:
+        print(f"Error parsing PDF: File not found at path: {file_path}")
+        return f"Error parsing PDF: File not found. Please check the file upload."
+    except PyPDF2.errors.PdfReadError as pre:
+        print(f"Error reading PDF: {pre}")
+        return f"Error reading PDF: The file might be corrupted or password-protected."
     except Exception as e:
         print(f"Error parsing PDF: {e}")
         return f"Error parsing PDF: {e}"
@@ -104,8 +102,9 @@ def Preprocess(textParty):
 def generate_summary(text):
     if not groq_client:
         return "Summarization is not available. Please set up your GROQ_API_KEY in the .env file."
-    if len(text) > 10000:
-        text = text[:10000]
+    # Adjusted truncation length for potentially better summary context
+    if len(text) > 15000:
+        text = text[:15000]
     try:
         completion = groq_client.chat.completions.create(
             model="llama3-8b-8192", # Or your preferred model
@@ -120,6 +119,61 @@ def generate_summary(text):
     except Exception as e:
         return f"Error generating summary: {str(e)}"
 
+# --- New LLM-based Search Function ---
+def get_contextual_search_result(target_word, tar_passage, groq_client_instance, max_context_length=8000):
+    """
+    Uses the LLM to provide contextual information about the target word within the passage.
+    """
+    if not target_word or target_word.strip() == "":
+        return "Please enter a search term."
+
+    if not groq_client_instance:
+        return "Contextual search requires the LLM API. Please set up your GROQ_API_KEY."
+
+    # Truncate passage if too long for the model/context window
+    original_length = len(tar_passage)
+    if original_length > max_context_length:
+        tar_passage_truncated = tar_passage[:max_context_length]
+        print(f"Warning: Passage truncated for LLM search context from {original_length} to {max_context_length} characters.")
+    else:
+        tar_passage_truncated = tar_passage
+
+    # --- Improved Prompt ---
+    prompt = f"""
+You are an expert political analyst. You have been given a section of a political manifesto and a specific search term.
+Your task is to extract and summarize all information related to the search term from the provided text.
+Focus on:
+1.  Specific policies, promises, or statements related to the term.
+2.  The context in which the term is used.
+3.  Any key details, figures, or commitments mentioned.
+Present your findings concisely. If the term is not relevant or not found in the provided text section, state that clearly.
+Search Term: {target_word}
+Manifesto Text Section:
+{tar_passage_truncated}
+Relevant Information:
+"""
+
+    try:
+        completion = groq_client_instance.chat.completions.create(
+            model="llama3-8b-8192", # Use the same or a suitable model
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant skilled at analyzing political texts and extracting relevant information based on a search query. Provide clear, concise summaries."},
+                {"role": "user", "content": prompt}
+            ],
+            temperature=0.2, # Low temperature for more factual extraction
+            max_tokens=1000  # Adjust based on expected output length
+        )
+        result = completion.choices[0].message.content.strip()
+        # Add a note if the input was truncated
+        if original_length > max_context_length:
+             result = f"(Note: Analysis based on the first {max_context_length} characters of the manifesto.)\n\n" + result
+        return result if result else f"No specific context for '{target_word}' could be generated from the provided text section."
+    except Exception as e:
+        error_msg = f"Error during contextual search for '{target_word}': {str(e)}"
+        print(error_msg)
+        traceback.print_exc()
+        return error_msg # Or return the error message directly
+
 def fDistance(text2Party):
     word_tokens_party = word_tokenize(text2Party)
     fdistance = FreqDist(word_tokens_party).most_common(10)
@@ -162,7 +216,7 @@ def normalize(d, target=1.0):
     return {key: value * factor for key, value in d.items()}
 
 # --- Visualization Functions with Error Handling ---
-
+# --- Improved safe_plot to handle apply_aspect errors ---
 def safe_plot(func, *args, **kwargs):
     """Executes a plotting function and returns the image, handling errors."""
     buf = None # Initialize buffer
@@ -195,21 +249,19 @@ def safe_plot(func, *args, **kwargs):
         plt.close('all') # Aggressive close on error
         return None
 
-
-
 def fDistancePlot(text2Party):
-    """Generates the frequency distribution plot."""
     def plot_func():
         tokens = word_tokenize(text2Party)
         if not tokens:
-            plt.text(0.5, 0.5, "No data to plot", ha='center', va='center', transform=plt.gca().transAxes) # Use Axes coordinates
-            return
+             plt.text(0.5, 0.5, "No data to plot", ha='center', va='center')
+             return
         fdist = FreqDist(tokens)
         fdist.plot(15, title='Frequency Distribution')
-        plt.xticks(rotation=45, ha='right')
+        plt.xticks(rotation=45, ha='right') # Rotate x-axis labels if needed
         plt.tight_layout()
     return safe_plot(plot_func)
 
+# --- Updated DispersionPlot without passing 'ax' ---
 def DispersionPlot(textParty):
     """Generates the word dispersion plot."""
     buf = None # Initialize buffer
@@ -232,7 +284,7 @@ def DispersionPlot(textParty):
             print("Warning: No common words found for dispersion plot.")
             return None
 
-        # --- Key Fix: Manage figure explicitly without passing 'ax' ---
+        # --- Manage figure explicitly without passing 'ax' ---
         fig = plt.figure(figsize=(10, 5)) # Create figure explicitly
         plt.title('Dispersion Plot')
         # Call dispersion_plot without 'ax' argument
@@ -264,16 +316,17 @@ def DispersionPlot(textParty):
         plt.close('all') # Aggressive close on error
         return None # Return None on error
 
-
+# --- Updated word_cloud_generator with robust figure handling ---
 def word_cloud_generator(parsed_text_name, text_Party):
     """Generates the word cloud image."""
     buf = None # Initialize buffer
     try:
+        # Handle case where parsed_text_name might not have .name
         filename_lower = ""
         if hasattr(parsed_text_name, 'name') and parsed_text_name.name:
             filename_lower = parsed_text_name.name.lower()
         elif isinstance(parsed_text_name, str):
-            filename_lower = parsed_text_name.lower()
+             filename_lower = parsed_text_name.lower()
 
         mask_path = None
         if 'bjp' in filename_lower:
@@ -283,16 +336,18 @@ def word_cloud_generator(parsed_text_name, text_Party):
         elif 'aap' in filename_lower:
             mask_path = 'aapMain2.jpg'
 
+        # Generate word cloud
         if text_Party.strip() == "":
-            raise ValueError("Text for word cloud is empty")
+             raise ValueError("Text for word cloud is empty")
 
         # Generate word cloud object
         if mask_path and os.path.exists(mask_path):
             orgImg = Image.open(mask_path)
+            # Ensure mask is in the right format (e.g., uint8)
             if orgImg.mode != 'RGB':
                 orgImg = orgImg.convert('RGB')
             mask = np.array(orgImg)
-            wordcloud = WordCloud(max_words=3000, mask=mask, background_color='white', mode='RGBA').generate(text_Party)
+            wordcloud = WordCloud(max_words=3000, mask=mask, background_color='white', mode='RGBA').generate(text_Party) # Added mode='RGBA'
         else:
             wordcloud = WordCloud(max_words=2000, background_color='white', mode='RGBA').generate(text_Party)
 
@@ -305,13 +360,13 @@ def word_cloud_generator(parsed_text_name, text_Party):
         buf = BytesIO()
         # Handle potential apply_aspect error for word cloud too
         try:
-             fig.savefig(buf, format='png', bbox_inches='tight', dpi=300, facecolor='white') # Added dpi and facecolor
+             fig.savefig(buf, format='png', bbox_inches='tight', dpi=150, facecolor='white') # Added dpi and facecolor
         except AttributeError as ae:
             if "apply_aspect" in str(ae):
                  print(f"Warning: bbox_inches='tight' failed for Word Cloud ({ae}), saving without it.")
                  buf.seek(0)
                  buf = BytesIO()
-                 fig.savefig(buf, format='png', dpi=300, facecolor='white')
+                 fig.savefig(buf, format='png', dpi=150, facecolor='white')
             else:
                  raise
         buf.seek(0)
@@ -327,109 +382,23 @@ def word_cloud_generator(parsed_text_name, text_Party):
         plt.close('all') # Aggressive close on error
         return None # Return None on error
 
-
-# Initial design for concordance based search
-def get_all_phases_containing_tar_wrd(target_word, tar_passage, left_margin=10, right_margin=10, numLins=4):
-    """
-    Function to get all the phrases that contain the target word in a text/passage.
-    """
-    if not target_word or target_word.strip() == "":
-        return "Please enter a search term"
-    tokens = nltk.word_tokenize(tar_passage)
-    text = nltk.Text(tokens)
-    c = nltk.ConcordanceIndex(text.tokens, key=lambda s: s.lower())
-    offsets = c.offsets(target_word)
-    if not offsets:
-         return f"Word '{target_word}' not found."
-    concordance_txt = [
-        text.tokens[max(0, offset - left_margin):offset + right_margin]
-        for offset in offsets[:numLins]
-    ]
-    result = [' '.join(con_sub) for con_sub in concordance_txt]
-    return '\n'.join(result) # Use newline for better readability in textbox
-
-
-def get_contextual_search_result(target_word, tar_passage, groq_client_instance, max_context_length=8000):
-    """
-    Uses the LLM to provide contextual information about the target word within the passage.
-    """
-    if not target_word or target_word.strip() == "":
-        return "Please enter a search term."
-
-    if not groq_client_instance:
-        return "Contextual search requires the LLM API. Please set up your GROQ_API_KEY."
-
-    # Basic check if word exists (optional, LLM can handle it too)
-    # Simple check, might generate false positives/negatives
-    # if target_word.lower() not in tar_passage.lower():
-    #     return f"The term '{target_word}' was not found in the manifesto text."
-
-    # Truncate passage if too long for the model/context window
-    original_length = len(tar_passage)
-    if original_length > max_context_length:
-        # Simple truncation; could be improved to ensure sentences are complete
-        tar_passage_truncated = tar_passage[:max_context_length]
-        print(f"Warning: Passage truncated for LLM search context from {original_length} to {max_context_length} characters.")
-    else:
-        tar_passage_truncated = tar_passage
-
-    # --- Improved Prompt ---
-    prompt = f"""
-You are an expert political analyst. You have been given a section of a political manifesto and a specific search term.
-Your task is to extract and summarize all information related to the search term from the provided text.
-Focus on:
-1.  Specific policies, promises, or statements related to the term.
-2.  The context in which the term is used.
-3.  Any key details, figures, or commitments mentioned.
-Present your findings concisely. If the term is not relevant or not found in the provided text section, state that clearly.
-Search Term: {target_word}
-Manifesto Text Section:
-{tar_passage_truncated}
-Relevant Information:
-"""
-
-    try:
-        completion = groq_client_instance.chat.completions.create(
-            model="llama3-8b-8192", # Use the same or a suitable model
-            messages=[
-                {"role": "system", "content": "You are a helpful assistant skilled at analyzing political texts and extracting relevant information based on a search query. Provide clear, concise summaries."},
-                {"role": "user", "content": prompt}
-            ],
-            temperature=0.2, # Low temperature for more factual extraction
-            max_tokens=1000  # Adjust based on expected output length
-        )
-        result = completion.choices[0].message.content.strip()
-        # Add a note if the input was truncated
-        if original_length > max_context_length:
-             result = f"(Note: Analysis based on the first {max_context_length} characters of the manifesto.)\n\n" + result
-        return result if result else f"No specific context for '{target_word}' could be generated from the provided text section."
-    except Exception as e:
-        error_msg = f"Error during contextual search for '{target_word}': {str(e)}"
-        print(error_msg)
-        traceback.print_exc()
-        # Fallback to concordance if LLM fails?
-        # return get_all_phases_containing_tar_wrd(target_word, tar_passage)
-        return error_msg # Or return the error message directly
-
-
+# --- Main Analysis Function ---
 def analysis(Manifesto, Search):
     try:
         if Manifesto is None:
-            # Ensure return order matches the outputs list
             return "No file uploaded", {}, None, None, None, None, None, "No file uploaded"
         if Search.strip() == "":
             Search = "government"
-        raw_party = Parsing(Manifesto)
+        raw_party = Parsing(Manifesto) # Uses PyPDF2 now
         if isinstance(raw_party, str) and raw_party.startswith("Error"):
             return raw_party, {}, None, None, None, None, None, "Parsing failed"
         text_Party = clean_text(raw_party)
         text_Party_processed = Preprocess(text_Party)
 
         # --- Perform Search FIRST using the ORIGINAL text for better context ---
-        # Pass the original raw text for richer context to the LLM
+        # Use the new LLM-based search function
         searChRes = get_contextual_search_result(Search, raw_party, groq_client)
 
-        # --- Then proceed with other analyses ---
         summary = generate_summary(raw_party) # Use raw_party for summary for more context?
 
         # --- Sentiment Analysis ---
@@ -450,9 +419,10 @@ def analysis(Manifesto, Search):
         sentiment_plot = safe_plot(lambda: df_dummy['Polarity_Label'].value_counts().plot(kind='bar', color="#FF9F45", title='Sentiment Analysis'))
         subjectivity_plot = safe_plot(lambda: df_dummy['Subjectivity_Label'].value_counts().plot(kind='bar', color="#B667F1", title='Subjectivity Analysis'))
         freq_plot = fDistancePlot(text_Party_processed)
-        dispersion_plot = DispersionPlot(text_Party_processed) # Use fixed version
-        wordcloud = word_cloud_generator(Manifesto, text_Party_processed) # Pass Manifesto object itself
+        dispersion_plot = DispersionPlot(text_Party_processed) # Uses updated version
+        wordcloud = word_cloud_generator(Manifesto, text_Party_processed) # Pass Manifesto object itself, uses updated version
         fdist_Party = fDistance(text_Party_processed)
+        # searChRes is now generated earlier using LLM
 
         return searChRes, fdist_Party, sentiment_plot, subjectivity_plot, wordcloud, freq_plot, dispersion_plot, summary
 
@@ -463,8 +433,8 @@ def analysis(Manifesto, Search):
         # Return error messages/images in the correct order
         return error_msg, {}, None, None, None, None, None, "Analysis failed"
 
-
-# --- Gradio Interface (remains largely the same, just ensuring output variable names match) ---
+# --- Gradio Interface ---
+# Use Blocks for custom layout
 with gr.Blocks(title='Manifesto Analysis') as demo:
     gr.Markdown("# Manifesto Analysis")
     # Input Section
@@ -481,9 +451,8 @@ with gr.Blocks(title='Manifesto Analysis') as demo:
         with gr.TabItem("Summary"):
             summary_output = gr.Textbox(label='AI-Generated Summary', lines=10, interactive=False)
 
-        # --- Search Results Tab ---
+        # --- Search Results Tab (uses LLM output now) ---
         with gr.TabItem("Search Results"):
-            # Use the specific output variable defined in the layout
             search_output = gr.Textbox(label='Context Based Search Results', lines=15, interactive=False, max_lines=20) # Increased lines/max_lines
 
         # --- Key Topics Tab ---
@@ -515,7 +484,7 @@ with gr.Blocks(title='Manifesto Analysis') as demo:
         fn=analysis,
         inputs=[file_input, search_input],
         outputs=[
-            search_output,        # 1 (Now contextual)
+            search_output,        # 1 (Now contextual LLM output)
             topics_output,        # 2
             sentiment_output,     # 3
             subjectivity_output,  # 4
@@ -528,7 +497,6 @@ with gr.Blocks(title='Manifesto Analysis') as demo:
     )
 
     # --- Examples ---
-    # Ensure outputs list references the PREDEFINED components from the layout
     gr.Examples(
         examples=[
             ["Example/AAP_Manifesto_2019.pdf", "government"],
@@ -536,11 +504,10 @@ with gr.Blocks(title='Manifesto Analysis') as demo:
             ["Example/Congress_Manifesto_2019.pdf", "safety"]
         ],
         inputs=[file_input, search_input],
-        # --- Key Fix: Reference the predefined output components ---
-        outputs=[search_output, topics_output, sentiment_output, subjectivity_output, wordcloud_output, freq_output, dispersion_output, summary_output],
+        outputs=[search_output, topics_output, sentiment_output, subjectivity_output, wordcloud_output, freq_output, dispersion_output, summary_output], # Link examples to outputs
         fn=analysis # Run analysis on example click
     )
 
-
+# Launch the app
 if __name__ == "__main__":
-    demo.launch(debug=True, share=False, show_error=True)
+    demo.launch(debug=True, share=False, show_error=True)