Update app.py

app.py

@@ -5,17 +5,10 @@ import streamlit as st
 from sentence_transformers import SentenceTransformer
 from scrapegraphai.graphs import SmartScraperMultiGraph
 from langchain_groq import ChatGroq
-import os
-from dotenv import load_dotenv
-import subprocess
-
-# Load environment variables
-load_dotenv()
-api_key = os.getenv("GROQ_API_KEY")
 
-# Install playwright
-subprocess.run(["playwright", "install"], check=True)
+import subprocess
 
+subprocess.run(["playwright", "install"])
 # Apply nest_asyncio to allow nested event loops
 nest_asyncio.apply()
 
@@ -26,9 +19,7 @@ model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 st.title("Course Scraper from Analytics Vidhya")
 
 # API Key Input
-if not api_key:
-    st.warning("GROQ_API_KEY not found. Please set it in your environment or enter it below.")
-    api_key = st.text_input("Enter your API Key:", type="password")
+api_key = st.text_input("Enter your API Key:", type="password")
 
 # Prompt Input
 user_prompt = st.text_input("Enter your prompt for scraping:")
@@ -51,41 +42,27 @@ graph_config = {
 base_url = "https://courses.analyticsvidhya.com/collections"
 urls = [f"{base_url}?page={i}" for i in range(1, 5)]  # Adjusting to scrape only the first 4 pages
 
-# Define the async scraping function
-async def scrape_courses():
+# Run the scraper when the button is clicked
+if st.button("Scrape Courses"):
     try:
-        # Create the SmartScraperMultiGraph instance
+        # Create the SmartScraperGraph instance
         smart_scraper_graph = SmartScraperMultiGraph(
             prompt=user_prompt,  # Use the user-defined prompt
             source=urls,
             config=graph_config
         )
-        # Run the scraper asynchronously
-        result = await smart_scraper_graph.run()
-        return result
-    except Exception as e:
-        st.error(f"An error occurred during scraping: {e}")
-        return None
 
-# Run the scraper when the button is clicked
-if st.button("Scrape Courses"):
-    if not user_prompt:
-        st.error("Please enter a prompt for scraping.")
-    elif not api_key:
-        st.error("Please enter a valid API key.")
-    else:
-        with st.spinner("Scraping in progress..."):
-            # Use asyncio to run the scraper
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            result = loop.run_until_complete(scrape_courses())
+        # Run the scraper
+        result = smart_scraper_graph.run()
+        
+        # Save the result as a JSON file
+        with open("courses.json", "w") as outfile:
+            json.dump(result, outfile, indent=4)
 
-        if result:
-            # Save the result as a JSON file
-            with open("courses.json", "w") as outfile:
-                json.dump(result, outfile, indent=4)
+        # Display the results in Streamlit
+        st.success("Scraping completed successfully!")
+        st.json(result)  # Display the result as a JSON object
 
-            # Display the results in Streamlit
-            st.success("Scraping completed successfully!")
-            st.json(result)  # Display the result as a JSON object
+    except Exception as e:
+        st.error(f"An error occurred: {e}")