Update 13_Qwen2_7b_finetuning_l40.ipynb

competition/13_Qwen2_7b_finetuning_l40.ipynb

@@ -14,7 +14,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": 2,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -27,7 +27,7 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "workding dir: c:\\Users\\HT\\Documents\\URP\\logical-reasoning\n"
+            "working dir: c:\\Users\\HT\\Documents\\URP\\logical-reasoning\n"
           ]
         }
       ],
@@ -44,7 +44,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 2,
+      "execution_count": 3,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -56,22 +56,22 @@
         {
           "data": {
             "text/plain": [
-              "False"
+              "True"
             ]
           },
-          "execution_count": 2,
+          "execution_count": 3,
           "metadata": {},
           "output_type": "execute_result"
         }
       ],
       "source": [
-        "need_to_setup_env = False\n",
+        "need_to_setup_env = True\n",
         "need_to_setup_env"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 3,
+      "execution_count": 4,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -79,7 +79,46 @@
         "id": "u0QXyHU_5DQR",
         "outputId": "54672b45-b5dc-48ef-efd2-5e8545e7b78b"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Looking in indexes: https://artifacts.forge.mastercard.com/artifactory/api/pypi/python/simple\n",
+            "Requirement already satisfied: huggingface_hub==0.23.2 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 1)) (0.23.2)\n",
+            "Requirement already satisfied: nltk==3.8.1 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 2)) (3.8.1)\n",
+            "Requirement already satisfied: python-dotenv==1.0.1 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 3)) (1.0.1)\n",
+            "Requirement already satisfied: black==24.4.0 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 4)) (24.4.0)\n",
+            "Requirement already satisfied: evaluate==0.4.2 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 5)) (0.4.2)\n",
+            "Requirement already satisfied: rouge_score==0.1.2 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 6)) (0.1.2)\n",
+            "Requirement already satisfied: pytest==8.2.1 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 7)) (8.2.1)\n",
+            "Requirement already satisfied: seaborn==0.13.2 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 8)) (0.13.2)\n",
+            "Requirement already satisfied: scikit-learn==1.5.0 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 9)) (1.5.0)\n",
+            "Requirement already satisfied: jupyter in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 10)) (1.0.0)\n",
+            "Requirement already satisfied: ipywidgets in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 11)) (8.1.3)\n",
+            "Requirement already satisfied: packaging in c:\\users\\ht\\appdata\\roaming\\python\\python312\\site-packages (from -r requirements.txt (line 12)) (24.1)\n",
+            "Requirement already satisfied: langchain_openai==0.1.13 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 13)) (0.1.13)\n",
+            "Requirement already satisfied: wandb==0.17.4 in c:\\users\\ht\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from -r requirements.txt (line 14)) (0.17.4)\n",
+            "Note: you may need to restart the kernel to use updated packages.\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x0000017A046EB3B0>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed')': /artifactory/api/pypi/python/simple/transformers/\n",
+            "WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x0000017A05F00EF0>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed')': /artifactory/api/pypi/python/simple/transformers/\n",
+            "WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x0000017A05F01820>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed')': /artifactory/api/pypi/python/simple/transformers/\n",
+            "WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x0000017A05F01220>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed')': /artifactory/api/pypi/python/simple/transformers/\n",
+            "WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x0000017A05F01A60>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed')': /artifactory/api/pypi/python/simple/transformers/\n",
+            "ERROR: Could not find a version that satisfies the requirement transformers==4.42.4 (from versions: none)\n",
+            "ERROR: No matching distribution found for transformers==4.42.4\n",
+            "\n",
+            "[notice] A new release of pip is available: 24.0 -> 24.1.2\n",
+            "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+          ]
+        }
+      ],
       "source": [
         "if need_to_setup_env:\n",
         "    %pip install -r requirements.txt"
@@ -87,7 +126,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 12,
+      "execution_count": 5,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -109,7 +148,7 @@
               "True"
             ]
           },
-          "execution_count": 12,
+          "execution_count": 5,
           "metadata": {},
           "output_type": "execute_result"
         }
@@ -127,7 +166,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 14,
+      "execution_count": 6,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -159,7 +198,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 19,
+      "execution_count": 7,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -271,7 +310,7 @@
               "4  真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季...  "
             ]
           },
-          "execution_count": 19,
+          "execution_count": 7,
           "metadata": {},
           "output_type": "execute_result"
         }
@@ -284,7 +323,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 20,
+      "execution_count": 8,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -301,7 +340,7 @@
               " 'output': '不是'}"
             ]
           },
-          "execution_count": 20,
+          "execution_count": 8,
           "metadata": {},
           "output_type": "execute_result"
         }
@@ -321,7 +360,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 9,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -334,7 +373,7 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "JSON file saved to /content/LLaMA-Factory/data/mgtv_train.json\n"
+            "JSON file saved to LLaMA-Factory/data/mgtv_train.json\n"
           ]
         }
       ],
@@ -343,7 +382,7 @@
         "import json\n",
         "\n",
         "# Define the directory where you want to save the JSON file\n",
-        "output_dir = \"/content/LLaMA-Factory/data/\"\n",
+        "output_dir = \"LLaMA-Factory/data/\"\n",
         "\n",
         "# Ensure the directory exists\n",
         "os.makedirs(output_dir, exist_ok=True)\n",
@@ -358,6 +397,208 @@
         "print(f\"JSON file saved to {json_file_path}\")\n"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Sat Jul 20 22:46:57 2024       \n",
+            "+---------------------------------------------------------------------------------------+\n",
+            "| NVIDIA-SMI 535.98                 Driver Version: 535.98       CUDA Version: 12.2     |\n",
+            "|-----------------------------------------+----------------------+----------------------+\n",
+            "| GPU  Name                     TCC/WDDM  | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
+            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
+            "|                                         |                      |               MIG M. |\n",
+            "|=========================================+======================+======================|\n",
+            "|   0  NVIDIA GeForce RTX 3070 ...  WDDM  | 00000000:01:00.0  On |                  N/A |\n",
+            "| N/A   41C    P8              15W / 140W |   1383MiB /  8192MiB |      2%      Default |\n",
+            "|                                         |                      |                  N/A |\n",
+            "+-----------------------------------------+----------------------+----------------------+\n",
+            "                                                                                         \n",
+            "+---------------------------------------------------------------------------------------+\n",
+            "| Processes:                                                                            |\n",
+            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
+            "|        ID   ID                                                             Usage      |\n",
+            "|=======================================================================================|\n",
+            "|    0   N/A  N/A       908      C   ...ta\\Local\\Programs\\Ollama\\ollama.exe    N/A      |\n",
+            "|    0   N/A  N/A      3092    C+G   ...n\\126.0.2592.102\\msedgewebview2.exe    N/A      |\n",
+            "|    0   N/A  N/A      7888    C+G   ...mpt_builder\\LogiAiPromptBuilder.exe    N/A      |\n",
+            "|    0   N/A  N/A     10552    C+G   ...tionsPlus\\logioptionsplus_agent.exe    N/A      |\n",
+            "|    0   N/A  N/A     10772    C+G   C:\\Windows\\explorer.exe                   N/A      |\n",
+            "|    0   N/A  N/A     12996    C+G   ....Search_cw5n1h2txyewy\\SearchApp.exe    N/A      |\n",
+            "|    0   N/A  N/A     15364    C+G   ...CBS_cw5n1h2txyewy\\TextInputHost.exe    N/A      |\n",
+            "|    0   N/A  N/A     15944    C+G   ...oogle\\Chrome\\Application\\chrome.exe    N/A      |\n",
+            "|    0   N/A  N/A     16984    C+G   ...t.LockApp_cw5n1h2txyewy\\LockApp.exe    N/A      |\n",
+            "|    0   N/A  N/A     19196    C+G   ...5n1h2txyewy\\ShellExperienceHost.exe    N/A      |\n",
+            "|    0   N/A  N/A     19352    C+G   ...ft Office\\root\\Office16\\ONENOTE.EXE    N/A      |\n",
+            "|    0   N/A  N/A     21732    C+G   ...aming\\Telegram Desktop\\Telegram.exe    N/A      |\n",
+            "|    0   N/A  N/A     22920    C+G   ...Desktop\\app-3.4.2\\GitHubDesktop.exe    N/A      |\n",
+            "|    0   N/A  N/A     27608    C+G   ...Programs\\Microsoft VS Code\\Code.exe    N/A      |\n",
+            "|    0   N/A  N/A     30412    C+G   ...8.0_x64__cv1g1gvanyjgm\\WhatsApp.exe    N/A      |\n",
+            "+---------------------------------------------------------------------------------------+\n"
+          ]
+        }
+      ],
+      "source": [
+        "!nvidia-smi"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 25,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Git Bash Path: C:/Program Files/Git/bin/bash.exe\n",
+            "Script Path: scripts/tune-lf.sh\n",
+            "Config Path: config/qwen2_7b_lora_sft_4bit.yaml\n",
+            "Config file not found at config/qwen2_7b_lora_sft_4bit.yaml\n"
+          ]
+        },
+        {
+          "ename": "CalledProcessError",
+          "evalue": "Command '['bash', 'scripts/tune-lf.sh', 'config/qwen2_7b_lora_sft_4bit.yaml']' returned non-zero exit status 127.",
+          "output_type": "error",
+          "traceback": [
+            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[1;31mCalledProcessError\u001b[0m                        Traceback (most recent call last)",
+            "File \u001b[1;32m<timed exec>:23\u001b[0m\n",
+            "File \u001b[1;32mc:\\Users\\HT\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\subprocess.py:571\u001b[0m, in \u001b[0;36mrun\u001b[1;34m(input, capture_output, timeout, check, *popenargs, **kwargs)\u001b[0m\n\u001b[0;32m    569\u001b[0m     retcode \u001b[38;5;241m=\u001b[39m process\u001b[38;5;241m.\u001b[39mpoll()\n\u001b[0;32m    570\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m check \u001b[38;5;129;01mand\u001b[39;00m retcode:\n\u001b[1;32m--> 571\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m CalledProcessError(retcode, process\u001b[38;5;241m.\u001b[39margs,\n\u001b[0;32m    572\u001b[0m                                  output\u001b[38;5;241m=\u001b[39mstdout, stderr\u001b[38;5;241m=\u001b[39mstderr)\n\u001b[0;32m    573\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m CompletedProcess(process\u001b[38;5;241m.\u001b[39margs, retcode, stdout, stderr)\n",
+            "\u001b[1;31mCalledProcessError\u001b[0m: Command '['bash', 'scripts/tune-lf.sh', 'config/qwen2_7b_lora_sft_4bit.yaml']' returned non-zero exit status 127."
+          ]
+        }
+      ],
+      "source": [
+        "%%time\n",
+        "import subprocess\n",
+        "import os\n",
+        "\n",
+        "git_bash_path = r\"C:/Program Files/Git/bin/bash.exe\"\n",
+        "\n",
+        "script_path = \"scripts/tune-lf.sh\"\n",
+        "config_path = \"config/qwen2_7b_lora_sft_4bit.yaml\"\n",
+        "\n",
+        "# Print paths for debugging\n",
+        "print(f\"Git Bash Path: {git_bash_path}\")\n",
+        "print(f\"Script Path: {script_path}\")\n",
+        "print(f\"Config Path: {config_path}\")\n",
+        "\n",
+        "# Check if the files exist\n",
+        "if not os.path.exists(git_bash_path):\n",
+        "    print(f\"Git Bash not found at {git_bash_path}\")\n",
+        "if not os.path.exists(script_path):\n",
+        "    print(f\"Script not found at {script_path}\")\n",
+        "if not os.path.exists(config_path):\n",
+        "    print(f\"Config file not found at {config_path}\")\n",
+        "\n",
+        "# Using Git Bash or WSL\n",
+        "subprocess.run(['bash', script_path, config_path], check=True)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 27,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Git Bash Path: C:/Program Files/Git/bin/bash.exe\n",
+            "Script Path: scripts/tune-lf.sh\n",
+            "Config Path: config/qwen2_7b_lora_sft_4bit.yaml\n",
+            "Config file not found at config/qwen2_7b_lora_sft_4bit.yaml\n",
+            "Command failed with error code: 127\n",
+            "Output: Current Directory:\n",
+            "/c/Users/HT/Documents/URP/logical-reasoning/llama-factory\n",
+            "config/qwen2_7b_lora_sft_4bit.yaml:\n",
+            " {\n",
+            "  \"model_name_or_path\": \"Qwen/Qwen2-7B\",\n",
+            "  \"stage\": \"sft\",\n",
+            "  \"do_train\": true,\n",
+            "  \"finetuning_type\": \"lora\",\n",
+            "  \"lora_target\": \"all\",\n",
+            "  \"quantization_bit\": 4,\n",
+            "  \"loraplus_lr_ratio\": 16.0,\n",
+            "  \"upcast_layernorm\": true,\n",
+            "  \"dataset\": \"mgtv_train\",\n",
+            "  \"template\": \"qwen\",\n",
+            "  \"cutoff_len\": 4096,\n",
+            "  \"max_samples\": 25000,\n",
+            "  \"overwrite_cache\": true,\n",
+            "  \"preprocessing_num_workers\": 16,\n",
+            "  \"output_dir\": \"saves/qwen2_7b/lora/sft_4bit\",\n",
+            "  \"logging_steps\": 562,\n",
+            "  \"save_steps\": 562,\n",
+            "  \"plot_loss\": true,\n",
+            "  \"overwrite_output_dir\": true,\n",
+            "  \"per_device_train_batch_size\": 32,\n",
+            "  \"gradient_accumulation_steps\": 8,\n",
+            "  \"learning_rate\": 0.0001,\n",
+            "  \"num_train_epochs\": 3.0,\n",
+            "  \"lr_scheduler_type\": \"cosine\",\n",
+            "  \"warmup_ratio\": 0.1,\n",
+            "  \"bf16\": true,\n",
+            "  \"ddp_timeout\": 180000000,\n",
+            "  \"val_size\": 0.1,\n",
+            "  \"per_device_eval_batch_size\": 1,\n",
+            "  \"eval_strategy\": \"steps\",\n",
+            "  \"eval_steps\": 562,\n",
+            "  \"report_to\": \"wandb\",\n",
+            "  \"run_name\": \"qwen2_7b\"\n",
+            "}\n",
+            "\n",
+            "Error: scripts/tune-lf.sh: line 9: llamafactory-cli: command not found\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "import subprocess\n",
+        "import os\n",
+        "\n",
+        "git_bash_path = r\"C:/Program Files/Git/bin/bash.exe\"\n",
+        "\n",
+        "script_path = \"scripts/tune-lf.sh\"\n",
+        "config_path = \"config/qwen2_7b_lora_sft_4bit.yaml\"\n",
+        "\n",
+        "# Print paths for debugging\n",
+        "print(f\"Git Bash Path: {git_bash_path}\")\n",
+        "print(f\"Script Path: {script_path}\")\n",
+        "print(f\"Config Path: {config_path}\")\n",
+        "\n",
+        "# Check if the files exist\n",
+        "if not os.path.exists(git_bash_path):\n",
+        "    print(f\"Git Bash not found at {git_bash_path}\")\n",
+        "if not os.path.exists(script_path):\n",
+        "    print(f\"Script not found at {script_path}\")\n",
+        "if not os.path.exists(config_path):\n",
+        "    print(f\"Config file not found at {config_path}\")\n",
+        "\n",
+        "# Ensure the script has execution permissions\n",
+        "os.chmod(script_path, 0o755)\n",
+        "\n",
+        "# Using Git Bash or WSL\n",
+        "try:\n",
+        "    result = subprocess.run(\n",
+        "        [git_bash_path, '-c', f'bash {script_path} {config_path}'], \n",
+        "        check=True, capture_output=True, text=True\n",
+        "    )\n",
+        "    print(\"Output:\", result.stdout)\n",
+        "    print(\"Error:\", result.stderr)\n",
+        "except subprocess.CalledProcessError as e:\n",
+        "    print(\"Command failed with error code:\", e.returncode)\n",
+        "    print(\"Output:\", e.stdout)\n",
+        "    print(\"Error:\", e.stderr)\n"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,