dtaa

Dockerfile

@@ -14,7 +14,7 @@ COPY requirements.txt .
 
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install --no-cache-dir wandb
+RUN pip install --no-cache-dir wandb tiktoken datasets
 
 # Create output directory with proper permissions
 RUN mkdir -p /app/out && chmod 777 /app/out
@@ -22,5 +22,10 @@ RUN mkdir -p /app/out && chmod 777 /app/out
 # Copy the project files
 COPY . .
 
+# Prepare the OpenWebText dataset
+RUN mkdir -p /app/data/openwebtext && \
+    cd /app/data/openwebtext && \
+    python prepare.py
+
 # Command to run training
 CMD ["python", "train.py", "--wandb_log=True"]