Update app.py

app.py

@@ -5,6 +5,9 @@ from huggingface_hub import whoami
 from huggingface_hub import HfApi
 is_spaces = True if os.environ.get("SPACE_ID") else False
 
+import tempfile
+from git import Repo
+
 if is_spaces:
     subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
     import spaces
@@ -280,6 +283,24 @@ def start_training(
             print(f"Added HF_TOKEN secret to {new_space_id}")
         except Exception as e:
             print(f"Failed to add HF_TOKEN secret to new space: {e}")
+
+            # Clone the subspace repo and push the dataset files into it
+            with tempfile.TemporaryDirectory() as clone_dir:
+                subspace_repo_url = f"https://{hf_token}@huggingface.co/spaces/{new_space_id}"
+                repo = Repo.clone_from(subspace_repo_url, clone_dir)
+
+                # Copy all dataset_folder contents to the clone dir
+                for item in os.listdir(dataset_folder):
+                    s = os.path.join(dataset_folder, item)
+                    d = os.path.join(clone_dir, item)
+                    if os.path.isdir(s):
+                        shutil.copytree(s, d, dirs_exist_ok=True)
+                    else:
+                        shutil.copy2(s, d)
+
+                repo.index.add(["."])
+                repo.index.commit("Push training files from controller")
+                repo.remote().push()
             
         cmd = f"autotrain spacerunner --project-name {slugged_lora_name} --script-path {dataset_folder}"
         cmd += f" --username {profile.username} --token {hf_token} --backend spaces-l4x1"