Update dataset_previews.py

dataset_previews.py

@@ -18,22 +18,21 @@ def calculate_dataset_statistics():
         test_df, val_df = load_mmlu_pro()
         
         test_df = test_df.sort_values(['category', 'question_id'])
-        print ("QUESTIONS TOT", len(test_df))
         
         all_subjects = sorted(test_df['category'].unique())
 
         # Calculate total questions and questions per subject
-        total_questions = 0
+        total_questions = len(test_df)
         subject_counts = {}
         
         # Count options per question
         options_counts = []
         
         for subject in all_subjects:
+            print("Subject", subject)
             test_samples = test_df[test_df['category'] == subject]
             num_questions = len(test_samples)
             subject_counts[subject] = num_questions
-            total_questions += num_questions
             print("Test_samples", test_samples[0], "\t Num Questions:", num_questions)
             
             # Count options for each question