Update app.py

app.py

@@ -7,12 +7,15 @@ from scrapegraphai.graphs import SmartScraperMultiGraph
 from langchain_groq import ChatGroq
 import os
 from dotenv import load_dotenv
+import subprocess
+
+# Load environment variables
 load_dotenv()
-api_key=os.getenv("GROQ_API_KEY")
+api_key = os.getenv("GROQ_API_KEY")
 
-import subprocess
+# Install playwright
+subprocess.run(["playwright", "install"], check=True)
 
-subprocess.run(["playwright", "install"])
 # Apply nest_asyncio to allow nested event loops
 nest_asyncio.apply()
 
@@ -23,7 +26,9 @@ model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 st.title("Course Scraper from Analytics Vidhya")
 
 # API Key Input
-#api_key = st.text_input("Enter your API Key:", type="password")
+if not api_key:
+    st.warning("GROQ_API_KEY not found. Please set it in your environment or enter it below.")
+    api_key = st.text_input("Enter your API Key:", type="password")
 
 # Prompt Input
 user_prompt = st.text_input("Enter your prompt for scraping:")
@@ -46,26 +51,41 @@ graph_config = {
 base_url = "https://courses.analyticsvidhya.com/collections"
 urls = [f"{base_url}?page={i}" for i in range(1, 5)]  # Adjusting to scrape only the first 4 pages
 
-# Run the scraper when the button is clicked
-if st.button("Scrape Courses"):
+# Define the async scraping function
+async def scrape_courses():
     try:
-        # Create the SmartScraperGraph instance
+        # Create the SmartScraperMultiGraph instance
         smart_scraper_graph = SmartScraperMultiGraph(
             prompt=user_prompt,  # Use the user-defined prompt
             source=urls,
             config=graph_config
         )
+        # Run the scraper asynchronously
+        result = await smart_scraper_graph.run()
+        return result
+    except Exception as e:
+        st.error(f"An error occurred during scraping: {e}")
+        return None
 
-        # Run the scraper
-        result = smart_scraper_graph.run()
-        
-        # Save the result as a JSON file
-        with open("courses.json", "w") as outfile:
-            json.dump(result, outfile, indent=4)
+# Run the scraper when the button is clicked
+if st.button("Scrape Courses"):
+    if not user_prompt:
+        st.error("Please enter a prompt for scraping.")
+    elif not api_key:
+        st.error("Please enter a valid API key.")
+    else:
+        with st.spinner("Scraping in progress..."):
+            # Use asyncio to run the scraper
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            result = loop.run_until_complete(scrape_courses())
 
-        # Display the results in Streamlit
-        st.success("Scraping completed successfully!")
-        st.json(result)  # Display the result as a JSON object
+        if result:
+            # Save the result as a JSON file
+            with open("courses.json", "w") as outfile:
+                json.dump(result, outfile, indent=4)
+
+            # Display the results in Streamlit
+            st.success("Scraping completed successfully!")
+            st.json(result)  # Display the result as a JSON object
 
-    except Exception as e:
-        st.error(f"An error occurred: {e}")