added the high and low roc value

.ipynb_checkpoints/distinguish_high_low_label-checkpoint.ipynb

@@ -0,0 +1,447 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "960bac80-51c7-4e9f-ad2d-84cd6c710f98",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pickle\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a34f21d0-0854-4a54-8f93-67718b2f969e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "file_path = \"roc_data2.pkl\"\n",
+    "\n",
+    "# Open and load the pickle file\n",
+    "with open(file_path, 'rb') as file:\n",
+    "    data = pickle.load(file)\n",
+    "\n",
+    "\n",
+    "# Print or use the data\n",
+    "# data[2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f9febed4-ce50-4e30-96ea-4b538ce2f9a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inc_slider=1\n",
+    "parent_location=\"ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/\"\n",
+    "test_info_location=parent_location+\"fullTest/test_info.txt\"\n",
+    "test_location=parent_location+\"fullTest/test.txt\"\n",
+    "test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')\n",
+    "grad_rate_data = pd.DataFrame(pd.read_pickle('school_grduation_rate.pkl'),columns=['school_number','grad_rate'])  # Load the grad_rate data\n",
+    "\n",
+    "# Step 1: Extract unique school numbers from test_info\n",
+    "unique_schools = test_info[0].unique()\n",
+    "\n",
+    "# Step 2: Filter the grad_rate_data using the unique school numbers\n",
+    "schools = grad_rate_data[grad_rate_data['school_number'].isin(unique_schools)]\n",
+    "\n",
+    "# Define a threshold for high and low graduation rates (adjust as needed)\n",
+    "grad_rate_threshold = 0.9  \n",
+    "\n",
+    "# Step 4: Divide schools into high and low graduation rate groups\n",
+    "high_grad_schools = schools[schools['grad_rate'] >= grad_rate_threshold]['school_number'].unique()\n",
+    "low_grad_schools = schools[schools['grad_rate'] < grad_rate_threshold]['school_number'].unique()\n",
+    "\n",
+    "# Step 5: Sample percentage of schools from each group\n",
+    "high_sample = pd.Series(high_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
+    "low_sample = pd.Series(low_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
+    "\n",
+    "# Step 6: Combine the sampled schools\n",
+    "random_schools = high_sample + low_sample\n",
+    "\n",
+    "# Step 7: Get indices for the sampled schools\n",
+    "indices = test_info[test_info[0].isin(random_schools)].index.tolist()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "fdfdf4b6-2752-4a21-9880-869af69f20cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()\n",
+    "low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "a79a4598-5702-4cc8-9f07-8e18fdda648b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "997"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(high_indices)+len(low_indices)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4707f3e6-2f44-46d8-ad8c-b6c244f693af",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>5342</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5343</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5344</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5345</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5346</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113359</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113360</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113361</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113362</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113363</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>997 rows × 1 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                        0\n",
+       "5342    PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...\n",
+       "5343    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "5344    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "5345    PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
+       "5346    PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...\n",
+       "...                                                   ...\n",
+       "113359  PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
+       "113360  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "113361  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "113362  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "113363  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "\n",
+       "[997 rows x 1 columns]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Load the test file and select rows based on indices\n",
+    "test = pd.read_csv(test_location, sep=',', header=None, engine='python')\n",
+    "selected_rows_df2 = test.loc[indices]\n",
+    "selected_rows_df2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "1d0c3d49-061f-486b-9c19-cf20945f3207",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "graduation_groups = [\n",
+    "    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index\n",
+    "]\n",
+    "# graduation_groups"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "ad0ce4a1-27fa-4867-8061-4054dbb340df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "t_label=data[0]\n",
+    "p_label=data[1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "id": "a4f4a2b9-3134-42ac-871b-4e117098cd0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 1: Align graduation_group, t_label, and p_label\n",
+    "aligned_labels = list(zip(graduation_groups, t_label, p_label))\n",
+    "\n",
+    "# Step 2: Separate the labels for high and low groups\n",
+    "high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']\n",
+    "low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']\n",
+    "\n",
+    "high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']\n",
+    "low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "id": "c8e34660-83d0-46a1-a218-95d609e11729",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "997"
+      ]
+     },
+     "execution_count": 50,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(low_t_labels)+len(high_t_labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "id": "c11050db-2636-4c50-9cd4-b9943e5cee83",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "id": "e1309e93-7063-4f48-bbc7-11a0d449c34e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ROC-AUC Score for High Graduation Rate Group: 0.675\n",
+      "ROC-AUC Score for Low Graduation Rate Group: 0.7489795918367347\n"
+     ]
+    }
+   ],
+   "source": [
+    "high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None\n",
+    "low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None\n",
+    "\n",
+    "print(\"ROC-AUC Score for High Graduation Rate Group:\", high_roc_auc)\n",
+    "print(\"ROC-AUC Score for Low Graduation Rate Group:\", low_roc_auc)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a99e7812-817d-4f9f-b6fa-1a58aa3a34dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "cannot convert the series to <class 'int'>",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[4], line 47\u001b[0m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(test_info_location, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m file:\n\u001b[0;32m     45\u001b[0m     data \u001b[38;5;241m=\u001b[39m file\u001b[38;5;241m.\u001b[39mreadlines()\n\u001b[1;32m---> 47\u001b[0m ideal_opt_task \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtest_info\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m7\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Assuming test_info[7] is accessible and holds the ideal task (1 or 2)\u001b[39;00m\n\u001b[0;32m     49\u001b[0m \u001b[38;5;66;03m# Initialize counters\u001b[39;00m\n\u001b[0;32m     50\u001b[0m task_counts \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m     51\u001b[0m     \u001b[38;5;241m1\u001b[39m: {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt1\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt2\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mboth\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m},\n\u001b[0;32m     52\u001b[0m     \u001b[38;5;241m2\u001b[39m: {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt1\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt2\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mboth\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m}\n\u001b[0;32m     53\u001b[0m }\n",
+      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\pandas\\core\\series.py:230\u001b[0m, in \u001b[0;36m_coerce_method.<locals>.wrapper\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    222\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    223\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCalling \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconverter\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on a single element Series is \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    224\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdeprecated and will raise a TypeError in the future. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    227\u001b[0m         stacklevel\u001b[38;5;241m=\u001b[39mfind_stack_level(),\n\u001b[0;32m    228\u001b[0m     )\n\u001b[0;32m    229\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m converter(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miloc[\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m--> 230\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot convert the series to \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconverter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
+      "\u001b[1;31mTypeError\u001b[0m: cannot convert the series to <class 'int'>"
+     ]
+    }
+   ],
+   "source": [
+    "parent_location=\"ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/\"\n",
+    "test_info_location=parent_location+\"fullTest/test_info.txt\"\n",
+    "test_location=parent_location+\"fullTest/test.txt\"\n",
+    "test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')\n",
+    "\n",
+    "def analyze_row(row, ideal_opt_task):\n",
+    "    # Split the row into fields\n",
+    "    fields = row.split(\"\\t\")\n",
+    "\n",
+    "    # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer\n",
+    "    optional_task_1_subtasks = [\"DenominatorFactor\", \"NumeratorFactor\", \"EquationAnswer\"]\n",
+    "    optional_task_2_subtasks = [\n",
+    "        \"FirstRow2:1\", \"FirstRow2:2\", \"FirstRow1:1\", \"FirstRow1:2\", \n",
+    "        \"SecondRow\", \"ThirdRow\"\n",
+    "    ]\n",
+    "    final_answer_tasks = [\"FinalAnswer\"]\n",
+    "\n",
+    "    # Helper function to evaluate task attempts\n",
+    "    def evaluate_tasks(fields, tasks):\n",
+    "        task_status = {}\n",
+    "        for task in tasks:\n",
+    "            relevant_attempts = [f for f in fields if task in f]\n",
+    "            if any(\"OK\" in attempt for attempt in relevant_attempts):\n",
+    "                task_status[task] = \"Attempted (Successful)\"\n",
+    "            elif any(\"ERROR\" in attempt for attempt in relevant_attempts):\n",
+    "                task_status[task] = \"Attempted (Error)\"\n",
+    "            elif any(\"JIT\" in attempt for attempt in relevant_attempts):\n",
+    "                task_status[task] = \"Attempted (JIT)\"\n",
+    "            else:\n",
+    "                task_status[task] = \"Unattempted\"\n",
+    "        return task_status\n",
+    "\n",
+    "    # Evaluate tasks for each category\n",
+    "    optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)\n",
+    "    optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)\n",
+    "\n",
+    "    # Check if tasks have any successful attempt\n",
+    "    opt1_done = any(status == \"Attempted (Successful)\" for status in optional_task_1_status.values())\n",
+    "    opt2_done = any(status == \"Attempted (Successful)\" for status in optional_task_2_status.values())\n",
+    "\n",
+    "    return opt1_done, opt2_done\n",
+    "\n",
+    "# Read data from test_info.txt\n",
+    "with open(test_info_location, \"r\") as file:\n",
+    "    data = file.readlines()\n",
+    "\n",
+    "ideal_opt_task = int(test_info[6])  # Assuming test_info[7] is accessible and holds the ideal task (1 or 2)\n",
+    "\n",
+    "# Initialize counters\n",
+    "task_counts = {\n",
+    "    1: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0},\n",
+    "    2: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0}\n",
+    "}\n",
+    "\n",
+    "for row in data:\n",
+    "    row = row.strip()\n",
+    "    if not row:\n",
+    "        continue\n",
+    "    opt1_done, opt2_done = analyze_row(row, ideal_opt_task)\n",
+    "\n",
+    "    if ideal_opt_task == 0:\n",
+    "        if opt1_done and not opt2_done:\n",
+    "            task_counts[1][\"only_opt1\"] += 1\n",
+    "        elif not opt1_done and opt2_done:\n",
+    "            task_counts[1][\"only_opt2\"] += 1\n",
+    "        elif opt1_done and opt2_done:\n",
+    "            task_counts[1][\"both\"] += 1\n",
+    "    elif ideal_opt_task == 1:\n",
+    "        if opt1_done and not opt2_done:\n",
+    "            task_counts[2][\"only_opt1\"] += 1\n",
+    "        elif not opt1_done and opt2_done:\n",
+    "            task_counts[2][\"only_opt2\"] += 1\n",
+    "        elif opt1_done and opt2_done:\n",
+    "            task_counts[2][\"both\"] += 1\n",
+    "\n",
+    "# Create a string output for results\n",
+    "output_summary = \"Task Analysis Summary:\\n\"\n",
+    "output_summary += \"-----------------------\\n\"\n",
+    "\n",
+    "for ideal_task, counts in task_counts.items():\n",
+    "    output_summary += f\"Ideal Task = OptionalTask_{ideal_task}:\\n\"\n",
+    "    output_summary += f\"  Only OptionalTask_1 done: {counts['only_opt1']}\\n\"\n",
+    "    output_summary += f\"  Only OptionalTask_2 done: {counts['only_opt2']}\\n\"\n",
+    "    output_summary += f\"  Both done: {counts['both']}\\n\"\n",
+    "\n",
+    "print(output_summary)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65ad9383-741f-44eb-8e8f-853ee7bc52a2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Untitled.ipynb

@@ -623,7 +623,7 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cu113:m122"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -637,7 +637,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.14"
+   "version": "3.12.4"
   }
  },
  "nbformat": 4,

app.py

@@ -8,6 +8,7 @@ import shutil
 import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
 import pandas as pd
+from sklearn.metrics import roc_auc_score
 # Define the function to process the input file and model selection
 
 def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
@@ -66,6 +67,8 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
 
     # Step 7: Get indices for the sampled schools
     indices = test_info[test_info[0].isin(random_schools)].index.tolist()
+    high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()
+    low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()
 
     # Load the test file and select rows based on indices
     test = pd.read_csv(test_location, sep=',', header=None, engine='python')
@@ -74,7 +77,27 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     # Save the selected rows to a file
     selected_rows_df2.to_csv('selected_rows.txt', sep='\t', index=False, header=False, quoting=3, escapechar=' ')
 
-   
+    graduation_groups = [
+    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index
+    ]
+
+    
+    with open("roc_data2.pkl", 'rb') as file:
+        data = pickle.load(file)
+    t_label=data[0]
+    p_label=data[1]
+    # Step 1: Align graduation_group, t_label, and p_label
+    aligned_labels = list(zip(graduation_groups, t_label, p_label))
+
+    # Step 2: Separate the labels for high and low groups
+    high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']
+    low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']
+
+    high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']
+    low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']
+
+    high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None
+    low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None
     # For demonstration purposes, we'll just return the content with the selected model name
 
     # print(checkpoint)
@@ -87,7 +110,7 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     #     model_name="highGRschool10"
     # Function to analyze each row
     def analyze_row(row):
-        # Split the row into fields
+    # Split the row into fields
         fields = row.split("\t")
 
         # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer
@@ -96,14 +119,12 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
             "FirstRow2:1", "FirstRow2:2", "FirstRow1:1", "FirstRow1:2", 
             "SecondRow", "ThirdRow"
         ]
-        final_answer_tasks = ["FinalAnswer"]
 
         # Helper function to evaluate task attempts
         def evaluate_tasks(fields, tasks):
             task_status = {}
             for task in tasks:
                 relevant_attempts = [f for f in fields if task in f]
-                # print(relevant_attempts)
                 if any("OK" in attempt for attempt in relevant_attempts):
                     task_status[task] = "Attempted (Successful)"
                 elif any("ERROR" in attempt for attempt in relevant_attempts):
@@ -117,40 +138,62 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
         # Evaluate tasks for each category
         optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)
         optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)
-        final_answer_status = evaluate_tasks(fields, final_answer_tasks)
-
-        # Combine results
-        result = {
-            "OptionalTask_1": optional_task_1_status,
-            "OptionalTask_2": optional_task_2_status,
-            "FinalAnswer": final_answer_status,
-        }
-        return result
+
+        # Check if tasks have any successful attempt
+        opt1_done = any(status == "Attempted (Successful)" for status in optional_task_1_status.values())
+        opt2_done = any(status == "Attempted (Successful)" for status in optional_task_2_status.values())
+
+        return opt1_done, opt2_done
+
+    # Read data from test_info.txt
     # Read data from test_info.txt
     with open(test_info_location, "r") as file:
         data = file.readlines()
-    results = [analyze_row(row.strip()) for row in data if row.strip()]
 
-    status_counts = {}
+    # Assuming test_info[7] is a list with ideal tasks for each instance
+    ideal_tasks = test_info[6]  # A list where each element is either 1 or 2
 
+    # Initialize counters
+    task_counts = {
+        1: {"only_opt1": 0, "only_opt2": 0, "both": 0},
+        2: {"only_opt1": 0, "only_opt2": 0, "both": 0}
+    }
 
-    for result in results:
-        for task_category, task_statuses in result.items():
-            for task, status in task_statuses.items():
-                if task not in status_counts:
-                    status_counts[task] = {"Attempted (Successful)": 0, "Attempted (Error)": 0, 
-                                        "Attempted (JIT)": 0, "Unattempted": 0}
-                status_counts[task][status] += 1
+    # Analyze rows
+    for i, row in enumerate(data):
+        row = row.strip()
+        if not row:
+            continue
+
+        ideal_task = ideal_tasks[i]  # Get the ideal task for the current row
+        opt1_done, opt2_done = analyze_row(row)
+
+        if ideal_task == 0:
+            if opt1_done and not opt2_done:
+                task_counts[1]["only_opt1"] += 1
+            elif not opt1_done and opt2_done:
+                task_counts[1]["only_opt2"] += 1
+            elif opt1_done and opt2_done:
+                task_counts[1]["both"] += 1
+        elif ideal_task == 1:
+            if opt1_done and not opt2_done:
+                task_counts[2]["only_opt1"] += 1
+            elif not opt1_done and opt2_done:
+                task_counts[2]["only_opt2"] += 1
+            elif opt1_done and opt2_done:
+                task_counts[2]["both"] += 1
 
     # Create a string output for results
     output_summary = "Task Analysis Summary:\n"
     output_summary += "-----------------------\n"
 
-    for task, statuses in status_counts.items():
-        output_summary += f"Task: {task}\n"
-        for status, count in statuses.items():
-            output_summary += f"  {status}: {count}\n"
+    for ideal_task, counts in task_counts.items():
+        output_summary += f"Ideal Task = OptionalTask_{ideal_task}:\n"
+        output_summary += f"  Only OptionalTask_1 done: {counts['only_opt1']}\n"
+        output_summary += f"  Only OptionalTask_2 done: {counts['only_opt2']}\n"
+        output_summary += f"  Both done: {counts['both']}\n"
 
+    # print(output_summary)
 
     progress(0.2, desc="analysis done!! Executing models")
     print("finetuned task: ",finetune_task)
@@ -175,10 +218,12 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
                 result[key]=value
             else:
                  result[key]=float(value)
+    result["ROC score of HGR"]=high_roc_auc
+    result["ROC score of LGR"]=low_roc_auc
 # Create a plot
     with open("roc_data.pkl", "rb") as f:
         fpr, tpr, _ = pickle.load(f)
-
+    # print(fpr,tpr)
     roc_auc = auc(fpr, tpr)
     fig, ax = plt.subplots()
     ax.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')
@@ -191,6 +236,10 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     plot_path = "plot.png"
     fig.savefig(plot_path)
     plt.close(fig)
+
+
+    
+
     progress(1.0)
     # Prepare text output
     text_output = f"Model: {model_name}\nResult:\n{result}"
@@ -203,9 +252,12 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     Total Schools in test: {len(unique_schools):.4f}\n
     Total number of instances having Schools with HGR : {len(high_sample):.4f}\n
     Total number of instances having Schools with LGR: {len(low_sample):.4f}\n
+
+    ROC score of HGR: {high_roc_auc}\n
+    ROC score of LGR: {low_roc_auc}\n
     -----------------\n
     """
-    return text_output,plot_path
+    return text_output,plot_path,output_summary
 
 # List of models for the dropdown menu
 
@@ -456,11 +508,11 @@ tbody.svelte-18wv37q>tr.svelte-18wv37q:nth-child(odd) {
     with gr.Row():
         output_text = gr.Textbox(label="")
         output_image = gr.Image(label="ROC")
-        # output_summary = gr.Textbox(label="Summary")
+        output_summary = gr.Textbox(label="Summary")
 
     btn = gr.Button("Submit")
  
-    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,output_image])
+    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,output_image,output_summary])
 
 
 # Launch the app

distinguish_high_low_label.ipynb

@@ -0,0 +1,451 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "960bac80-51c7-4e9f-ad2d-84cd6c710f98",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pickle\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a34f21d0-0854-4a54-8f93-67718b2f969e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "file_path = \"roc_data2.pkl\"\n",
+    "\n",
+    "# Open and load the pickle file\n",
+    "with open(file_path, 'rb') as file:\n",
+    "    data = pickle.load(file)\n",
+    "\n",
+    "\n",
+    "# Print or use the data\n",
+    "# data[2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f9febed4-ce50-4e30-96ea-4b538ce2f9a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inc_slider=1\n",
+    "parent_location=\"ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/\"\n",
+    "test_info_location=parent_location+\"fullTest/test_info.txt\"\n",
+    "test_location=parent_location+\"fullTest/test.txt\"\n",
+    "test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')\n",
+    "grad_rate_data = pd.DataFrame(pd.read_pickle('school_grduation_rate.pkl'),columns=['school_number','grad_rate'])  # Load the grad_rate data\n",
+    "\n",
+    "# Step 1: Extract unique school numbers from test_info\n",
+    "unique_schools = test_info[0].unique()\n",
+    "\n",
+    "# Step 2: Filter the grad_rate_data using the unique school numbers\n",
+    "schools = grad_rate_data[grad_rate_data['school_number'].isin(unique_schools)]\n",
+    "\n",
+    "# Define a threshold for high and low graduation rates (adjust as needed)\n",
+    "grad_rate_threshold = 0.9  \n",
+    "\n",
+    "# Step 4: Divide schools into high and low graduation rate groups\n",
+    "high_grad_schools = schools[schools['grad_rate'] >= grad_rate_threshold]['school_number'].unique()\n",
+    "low_grad_schools = schools[schools['grad_rate'] < grad_rate_threshold]['school_number'].unique()\n",
+    "\n",
+    "# Step 5: Sample percentage of schools from each group\n",
+    "high_sample = pd.Series(high_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
+    "low_sample = pd.Series(low_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
+    "\n",
+    "# Step 6: Combine the sampled schools\n",
+    "random_schools = high_sample + low_sample\n",
+    "\n",
+    "# Step 7: Get indices for the sampled schools\n",
+    "indices = test_info[test_info[0].isin(random_schools)].index.tolist()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "fdfdf4b6-2752-4a21-9880-869af69f20cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()\n",
+    "low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "a79a4598-5702-4cc8-9f07-8e18fdda648b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "997"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(high_indices)+len(low_indices)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4707f3e6-2f44-46d8-ad8c-b6c244f693af",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>5342</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5343</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5344</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5345</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5346</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113359</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113360</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113361</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113362</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>113363</th>\n",
+       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>997 rows × 1 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                        0\n",
+       "5342    PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...\n",
+       "5343    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "5344    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "5345    PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
+       "5346    PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...\n",
+       "...                                                   ...\n",
+       "113359  PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
+       "113360  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "113361  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "113362  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "113363  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
+       "\n",
+       "[997 rows x 1 columns]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Load the test file and select rows based on indices\n",
+    "test = pd.read_csv(test_location, sep=',', header=None, engine='python')\n",
+    "selected_rows_df2 = test.loc[indices]\n",
+    "selected_rows_df2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "1d0c3d49-061f-486b-9c19-cf20945f3207",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "graduation_groups = [\n",
+    "    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index\n",
+    "]\n",
+    "# graduation_groups"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "ad0ce4a1-27fa-4867-8061-4054dbb340df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "t_label=data[0]\n",
+    "p_label=data[1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "id": "a4f4a2b9-3134-42ac-871b-4e117098cd0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 1: Align graduation_group, t_label, and p_label\n",
+    "aligned_labels = list(zip(graduation_groups, t_label, p_label))\n",
+    "\n",
+    "# Step 2: Separate the labels for high and low groups\n",
+    "high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']\n",
+    "low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']\n",
+    "\n",
+    "high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']\n",
+    "low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "id": "c8e34660-83d0-46a1-a218-95d609e11729",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "997"
+      ]
+     },
+     "execution_count": 50,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(low_t_labels)+len(high_t_labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "id": "c11050db-2636-4c50-9cd4-b9943e5cee83",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "id": "e1309e93-7063-4f48-bbc7-11a0d449c34e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ROC-AUC Score for High Graduation Rate Group: 0.675\n",
+      "ROC-AUC Score for Low Graduation Rate Group: 0.7489795918367347\n"
+     ]
+    }
+   ],
+   "source": [
+    "high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None\n",
+    "low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None\n",
+    "\n",
+    "print(\"ROC-AUC Score for High Graduation Rate Group:\", high_roc_auc)\n",
+    "print(\"ROC-AUC Score for Low Graduation Rate Group:\", low_roc_auc)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "a99e7812-817d-4f9f-b6fa-1a58aa3a34dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Task Analysis Summary:\n",
+      "-----------------------\n",
+      "Ideal Task = OptionalTask_1:\n",
+      "  Only OptionalTask_1 done: 22501\n",
+      "  Only OptionalTask_2 done: 20014\n",
+      "  Both done: 24854\n",
+      "Ideal Task = OptionalTask_2:\n",
+      "  Only OptionalTask_1 done: 12588\n",
+      "  Only OptionalTask_2 done: 18942\n",
+      "  Both done: 15147\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def analyze_row(row):\n",
+    "    # Split the row into fields\n",
+    "    fields = row.split(\"\\t\")\n",
+    "\n",
+    "    # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer\n",
+    "    optional_task_1_subtasks = [\"DenominatorFactor\", \"NumeratorFactor\", \"EquationAnswer\"]\n",
+    "    optional_task_2_subtasks = [\n",
+    "        \"FirstRow2:1\", \"FirstRow2:2\", \"FirstRow1:1\", \"FirstRow1:2\", \n",
+    "        \"SecondRow\", \"ThirdRow\"\n",
+    "    ]\n",
+    "\n",
+    "    # Helper function to evaluate task attempts\n",
+    "    def evaluate_tasks(fields, tasks):\n",
+    "        task_status = {}\n",
+    "        for task in tasks:\n",
+    "            relevant_attempts = [f for f in fields if task in f]\n",
+    "            if any(\"OK\" in attempt for attempt in relevant_attempts):\n",
+    "                task_status[task] = \"Attempted (Successful)\"\n",
+    "            elif any(\"ERROR\" in attempt for attempt in relevant_attempts):\n",
+    "                task_status[task] = \"Attempted (Error)\"\n",
+    "            elif any(\"JIT\" in attempt for attempt in relevant_attempts):\n",
+    "                task_status[task] = \"Attempted (JIT)\"\n",
+    "            else:\n",
+    "                task_status[task] = \"Unattempted\"\n",
+    "        return task_status\n",
+    "\n",
+    "    # Evaluate tasks for each category\n",
+    "    optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)\n",
+    "    optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)\n",
+    "\n",
+    "    # Check if tasks have any successful attempt\n",
+    "    opt1_done = any(status == \"Attempted (Successful)\" for status in optional_task_1_status.values())\n",
+    "    opt2_done = any(status == \"Attempted (Successful)\" for status in optional_task_2_status.values())\n",
+    "\n",
+    "    return opt1_done, opt2_done\n",
+    "\n",
+    "# Read data from test_info.txt\n",
+    "# Read data from test_info.txt\n",
+    "with open(test_info_location, \"r\") as file:\n",
+    "    data = file.readlines()\n",
+    "\n",
+    "# Assuming test_info[7] is a list with ideal tasks for each instance\n",
+    "ideal_tasks = test_info[6]  # A list where each element is either 1 or 2\n",
+    "\n",
+    "# Initialize counters\n",
+    "task_counts = {\n",
+    "    1: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0},\n",
+    "    2: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0}\n",
+    "}\n",
+    "\n",
+    "# Analyze rows\n",
+    "for i, row in enumerate(data):\n",
+    "    row = row.strip()\n",
+    "    if not row:\n",
+    "        continue\n",
+    "\n",
+    "    ideal_task = ideal_tasks[i]  # Get the ideal task for the current row\n",
+    "    opt1_done, opt2_done = analyze_row(row)\n",
+    "\n",
+    "    if ideal_task == 0:\n",
+    "        if opt1_done and not opt2_done:\n",
+    "            task_counts[1][\"only_opt1\"] += 1\n",
+    "        elif not opt1_done and opt2_done:\n",
+    "            task_counts[1][\"only_opt2\"] += 1\n",
+    "        elif opt1_done and opt2_done:\n",
+    "            task_counts[1][\"both\"] += 1\n",
+    "    elif ideal_task == 1:\n",
+    "        if opt1_done and not opt2_done:\n",
+    "            task_counts[2][\"only_opt1\"] += 1\n",
+    "        elif not opt1_done and opt2_done:\n",
+    "            task_counts[2][\"only_opt2\"] += 1\n",
+    "        elif opt1_done and opt2_done:\n",
+    "            task_counts[2][\"both\"] += 1\n",
+    "\n",
+    "# Create a string output for results\n",
+    "output_summary = \"Task Analysis Summary:\\n\"\n",
+    "output_summary += \"-----------------------\\n\"\n",
+    "\n",
+    "for ideal_task, counts in task_counts.items():\n",
+    "    output_summary += f\"Ideal Task = OptionalTask_{ideal_task}:\\n\"\n",
+    "    output_summary += f\"  Only OptionalTask_1 done: {counts['only_opt1']}\\n\"\n",
+    "    output_summary += f\"  Only OptionalTask_2 done: {counts['only_opt2']}\\n\"\n",
+    "    output_summary += f\"  Both done: {counts['both']}\\n\"\n",
+    "\n",
+    "print(output_summary)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65ad9383-741f-44eb-8e8f-853ee7bc52a2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

new_test_saved_finetuned_model.py

@@ -221,9 +221,12 @@ class BERTFineTuneTrainer:
                 for key, value in final_msg.items():
                     file.write(f"{key}: {value}\n")
             print(final_msg)
+            # print(type(plabels),type(tlabels),plabels,tlabels) 
             fpr, tpr, thresholds = roc_curve(tlabels, positive_class_probs)
             with open("roc_data.pkl", "wb") as f:
                 pickle.dump((fpr, tpr, thresholds), f)
+            with open("roc_data2.pkl", "wb") as f:
+                pickle.dump((tlabels,positive_class_probs), f)                
             print(final_msg)
             f.close()
             with open(self.log_folder_path+f"/log_{phase}_finetuned_info.txt", 'a') as f1:
@@ -426,6 +429,7 @@ class BERTFineTuneCalibratedTrainer:
             auc_score = roc_auc_score(tlabels, positive_class_probs)
             end_time = time.time()
             final_msg = {
+                "this one":"this one",
                 "avg_loss": avg_loss / len(data_iter),
                 "total_acc": total_correct * 100.0 / total_element,
                 "precisions": precisions,
@@ -441,8 +445,7 @@ class BERTFineTuneCalibratedTrainer:
                 for key, value in final_msg.items():
                     file.write(f"{key}: {value}\n")
             with open("plabels.txt","w") as file:
-                file.write(plabels)
-                       
+                file.write(plabels)          
             print(final_msg)
             fpr, tpr, thresholds = roc_curve(tlabels, positive_class_probs)
             f.close()

result.txt

@@ -3,5 +3,5 @@ total_acc: 69.00702106318957
 precisions: 0.7236623191454734
 recalls: 0.6900702106318957
 f1_scores: 0.6802420656474512
-time_taken_from_start: 30.22950553894043
+time_taken_from_start: 21.604072332382202
 auc_score: 0.7457100293916334

roc_data2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fa9d96833c12979f8495141ee61c0ba07d4a20c5fb5bc18a7f72bf4d15e8fd
+size 28023