Upload llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256

llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/bench.slurm

@@ -1,16 +1,16 @@
 #!/bin/bash
 
 #SBATCH --job-name=bench_cluster
-#SBATCH --time=00:59:00
+#SBATCH --time=01:30:00
 #SBATCH --partition=hopper-prod
 #SBATCH --nodes=2
 #SBATCH --gres=gpu:8
-#SBATCH --qos=high
+#SBATCH --qos=normal
 #SBATCH --ntasks-per-node=1
 #SBATCH --cpus-per-task=96
 #SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out
+#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out
+#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out
 
 # Function to update status based on squeue output
 update_status() {
@@ -53,7 +53,7 @@ huggingface-cli login --token $HUGGINGFACE_TOKEN
 
 
 NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/config.yaml"
+CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/config.yaml"
 
 LAUNCHER="torchrun \
    --nproc_per_node 8 \
@@ -72,7 +72,7 @@ cd ..
 job_id=${SLURM_JOB_ID}
 
 # Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt &
+update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt &
 
 # Run the main command
 srun -u $LAUNCHER $CMD
@@ -80,28 +80,28 @@ exit_status=$?
 
 # Update status based on the exit status of `srun`
 if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
+    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
 else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
+    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
+    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
+    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/log.out; then
+        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
     else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
+        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt
     fi 
 fi
 
 # Run the report script if the job completed successfully
 if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 --is_profiler
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 --is_logs
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 --is_profiler
 fi
 
 
 # Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 --commit-message "Upload llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256"
+huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256 --commit-message "Upload llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256"
 
 # Verify the upload
 if [ $? -eq 0 ]; then

llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-256/status.txt

@@ -1 +1 @@
-fail
+oom