Update app.py

app.py

@@ -5,7 +5,7 @@ import zipfile
 from datasets import Dataset
 from huggingface_hub import HfApi
 import logging
-from datetime import datetime
+import time  # Import time module for adding delays
 
 # Set up logging
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -55,7 +55,7 @@ def segment_text(file_path):
     logger.info(f"Segmented text into {len(chunks)} chunks.")
     return chunks
 
-# Function to process the text file and make API calls
+# Function to process the text file and make API calls with rate limiting
 def process_text(file, prompt):
     try:
         logger.info("Starting text processing...")
@@ -64,40 +64,48 @@ def process_text(file, prompt):
         file_path = file.name if hasattr(file, "name") else file
         chunks = segment_text(file_path)
         
-        # Perform API calls for each chunk
+        # Initialize Hugging Face API
+        hf_api = HfApi(token=os.environ.get("HUGGINGFACE_TOKEN"))
+        if not hf_api.token:
+            raise ValueError("Hugging Face token not found in environment variables.")
+        
+        # Process each chunk with a 20-second delay between API calls
         results = []
         for idx, chunk in enumerate(chunks):
             logger.info(f"Processing chunk {idx + 1}/{len(chunks)}")
             try:
+                # Call the API
                 result = call_api(f"{prompt}\n\n{chunk}")
                 results.append(result)
                 logger.info(f"Chunk {idx + 1} processed successfully.")
+                
+                # Save the result to a file
+                os.makedirs("outputs", exist_ok=True)
+                output_file = f"outputs/output_{idx}.txt"
+                with open(output_file, "w", encoding="utf-8") as f:
+                    f.write(result)
+                logger.info(f"Saved result to {output_file}")
+                
+                # Upload the chunk to Hugging Face
+                try:
+                    logger.info(f"Uploading chunk {idx + 1} to Hugging Face...")
+                    dataset = Dataset.from_dict({"text": [result]})
+                    dataset.push_to_hub("TeacherPuffy/book")  # Updated dataset name
+                    logger.info(f"Chunk {idx + 1} uploaded to Hugging Face successfully.")
+                except Exception as e:
+                    logger.error(f"Failed to upload chunk {idx + 1} to Hugging Face: {e}")
+                    raise gr.Error(f"Failed to upload chunk {idx + 1} to Hugging Face: {str(e)}")
+                
+                # Wait 20 seconds before the next API call
+                if idx < len(chunks) - 1:  # No need to wait after the last chunk
+                    logger.info("Waiting 20 seconds before the next API call...")
+                    time.sleep(20)
+                
             except Exception as e:
                 logger.error(f"Failed to process chunk {idx + 1}: {e}")
                 raise gr.Error(f"Failed to process chunk {idx + 1}: {str(e)}")
         
-        # Save results as individual text files
-        os.makedirs("outputs", exist_ok=True)
-        for idx, result in enumerate(results):
-            output_file = f"outputs/output_{idx}.txt"
-            with open(output_file, "w", encoding="utf-8") as f:
-                f.write(result)
-            logger.info(f"Saved result to {output_file}")
-        
-        # Upload to Hugging Face dataset
-        try:
-            logger.info("Uploading results to Hugging Face dataset...")
-            hf_api = HfApi(token=os.environ.get("HUGGINGFACE_TOKEN"))
-            if not hf_api.token:
-                raise ValueError("Hugging Face token not found in environment variables.")
-            dataset = Dataset.from_dict({"text": results})
-            dataset.push_to_hub("TeacherPuffy/book")  # Updated dataset name
-            logger.info("Results uploaded to Hugging Face dataset successfully.")
-        except Exception as e:
-            logger.error(f"Failed to upload to Hugging Face: {e}")
-            raise gr.Error(f"Failed to upload to Hugging Face: {str(e)}")
-        
-        # Create a ZIP file
+        # Create a ZIP file of all outputs
         try:
             logger.info("Creating ZIP file...")
             with zipfile.ZipFile("outputs.zip", "w") as zipf:
@@ -109,7 +117,7 @@ def process_text(file, prompt):
             logger.error(f"Failed to create ZIP file: {e}")
             raise gr.Error(f"Failed to create ZIP file: {str(e)}")
         
-        return "outputs.zip", "Results uploaded to Hugging Face dataset and ZIP file created."
+        return "outputs.zip", "All chunks processed and uploaded to Hugging Face. ZIP file created."
     
     except Exception as e:
         logger.error(f"An error occurred during processing: {e}")
@@ -117,7 +125,7 @@ def process_text(file, prompt):
 
 # Gradio interface
 with gr.Blocks() as demo:
-    gr.Markdown("## Text File Processor with API Calls")
+    gr.Markdown("## Text File Processor with Rate-Limited API Calls")
     with gr.Row():
         file_input = gr.File(label="Upload Text File")
         prompt_input = gr.Textbox(label="Enter Prompt")