deepseek-reasoner prompt engineering and UI improvements (#8)

- deepseek-reasoner prompt engineering and UI improvements (806eaf0e3d06715c3aae050362fc88852f203145)


Co-authored-by: Furkan Eris <[email protected]>

app.py

@@ -230,14 +230,22 @@ class PAS2:
             model_id = model_config["model_id"]
             model_type = model_config["type"]
             
+            # Customize messages based on model
+            system_content = "You are a helpful AI assistant. Provide accurate, factual information in response to questions."
+            user_content = query
+            
+            # Special handling for deepseek-reasoner
+            if model_id == "deepseek-reasoner":
+                user_content = f"Extract the following information and format it as JSON:\n\n{query}"
+                
             messages = [
                 {
                     "role": "system",
-                    "content": "You are a helpful AI assistant. Provide accurate, factual information in response to questions."
+                    "content": system_content
                 },
                 {
                     "role": "user",
-                    "content": query
+                    "content": user_content
                 }
             ]
             
@@ -450,27 +458,49 @@ Your response should be a JSON with the following fields:
         try:
             logger.info("Sending judgment request to %s...", self.judge_model)
             
+            # Customize the system prompt for deepseek-reasoner
+            customized_system_prompt = system_prompt
+            user_content = f"Evaluate these responses for hallucinations:\n\n{context}"
+            
+            # Additional prompt engineering for deepseek-reasoner
+            if model_id == "deepseek-reasoner":
+                user_content = f"""Extract the following information and format it as JSON:
+
+Evaluate these responses for hallucinations:\n\n{context}\n\n
+                - hallucination_detected: boolean indicating whether hallucinations were found
+                - confidence_score: number between 0 and 1 representing your confidence in the judgment
+                - conflicting_facts: an array of objects describing any conflicting information found
+                - reasoning: detailed explanation for your judgment
+                - summary: a concise summary of your analysis
+                
+                Respond ONLY with valid JSON and no other text.
+                """
+            
             # Use the appropriate client and model based on the type
             if model_type == "mistral":
                 response = client.chat.complete(
                     model=model_id,
                     messages=[
-                        {"role": "system", "content": system_prompt},
-                        {"role": "user", "content": f"Evaluate these responses for hallucinations:\n\n{context}"}
+                        {"role": "system", "content": customized_system_prompt},
+                        {"role": "user", "content": user_content}
                     ],
                     response_format={"type": "json_object"}
                 )
-                result_json = json.loads(response.choices[0].message.content)
+                content = response.choices[0].message.content
+                # Normal JSON parsing for mistral
+                result_json = json.loads(content)
             else:  # openai-compatible API
                 response = client.chat.completions.create(
                     model=model_id,
                     messages=[
-                        {"role": "system", "content": system_prompt},
-                        {"role": "user", "content": f"Evaluate these responses for hallucinations:\n\n{context}"}
+                        {"role": "system", "content": customized_system_prompt},
+                        {"role": "user", "content": user_content}
                     ],
                     response_format={"type": "json_object"}
                 )
-                result_json = json.loads(response.choices[0].message.content)
+                content = response.choices[0].message.content
+                
+                result_json = json.loads(content)
             
             logger.debug("Received judgment response from %s: %s", self.judge_model, result_json)
             
@@ -1288,36 +1318,62 @@ def create_interface():
     .info-box {
         padding: 1.2em;
         border-radius: 8px;
-        background-color: #b0bec5;
+        background-color: #eceff1;
         margin-bottom: 1em;
         box-shadow: 0 2px 5px rgba(0,0,0,0.05);
+        color: #455a64;
+        line-height: 1.5;
+        border-left: 3px solid #607d8b;
+    }
+    .info-box p strong {
         color: #263238;
+        font-weight: 600;
     }
     .hallucination-positive {
         padding: 1.2em;
         border-radius: 8px;
-        background-color: #ffcdd2;
-        border-left: 5px solid #d32f2f;
+        background-color: #f8e8e8;
+        border-left: 5px solid #c62828;
         margin-bottom: 1em;
         box-shadow: 0 2px 5px rgba(0,0,0,0.05);
-        color: #b71c1c;
+        color: #4d0c0c;
+    }
+    .hallucination-positive h3 {
+        color: #c62828;
+        margin-top: 0;
+        margin-bottom: 0.5em;
+    }
+    .hallucination-positive p {
+        color: #5d4141;
+        line-height: 1.5;
     }
     .hallucination-negative {
         padding: 1.2em;
         border-radius: 8px;
-        background-color: #c8e6c9;
-        border-left: 5px solid #388e3c;
+        background-color: #e8f5e9;
+        border-left: 5px solid #2e7d32;
         margin-bottom: 1em;
         box-shadow: 0 2px 5px rgba(0,0,0,0.05);
         color: #1b5e20;
     }
+    .hallucination-negative h3 {
+        color: #2e7d32;
+        margin-top: 0;
+        margin-bottom: 0.5em;
+    }
+    .hallucination-negative p {
+        color: #3e5e40;
+        line-height: 1.5;
+    }
     .response-box {
         padding: 1.2em;
         border-radius: 8px;
-        background-color: #b0bec5;
+        background-color: #eceff1;
         margin-bottom: 0.8em;
         box-shadow: 0 2px 5px rgba(0,0,0,0.05);
-        color: #263238;
+        color: #37474f;
+        line-height: 1.5;
+        border-left: 3px solid #78909c;
     }
     .example-queries {
         display: flex;
@@ -1801,11 +1857,19 @@ def create_interface():
         
         response = detector.save_feedback(results, combined_feedback)
         
-        # Return a success message that will trigger a JS notification
-        feedback_response = """
+        # Check if this is a duplicate feedback submission message
+        is_duplicate = "already provided feedback" in response
+        notification_color = "#ff9800" if is_duplicate else "#4caf50"
+        icon = "ℹ" if is_duplicate else "✓"
+        heading_text = "Note" if is_duplicate else "Thank You!"
+        message_text = response
+        status_text = "already submitted" if is_duplicate else "submitted successfully"
+        
+        # Return a message that will trigger a JS notification
+        feedback_response = f"""
         <div id="feedback-popup-container"></div>
         <script>
-        (function() {
+        (function() {{
             // Create the notification element
             const container = document.getElementById('feedback-popup-container');
             const notification = document.createElement('div');
@@ -1814,7 +1878,7 @@ def create_interface():
                 position: fixed;
                 top: 50px;
                 right: 20px;
-                background-color: #4caf50;
+                background-color: {notification_color};
                 color: white;
                 padding: 15px;
                 border-radius: 5px;
@@ -1830,16 +1894,18 @@ def create_interface():
             // Create notification content
             const checkmark = document.createElement('div');
             checkmark.style.marginRight = '10px';
-            checkmark.textContent = '✓';
+            checkmark.textContent = '{icon}';
             
             const textContainer = document.createElement('div');
             
             const heading = document.createElement('div');
             heading.style.fontWeight = 'bold';
-            heading.textContent = 'Thank You!';
+            heading.textContent = '{heading_text}';
             
             const message = document.createElement('div');
-            message.textContent = 'Your feedback has been recorded.';
+            message.textContent = '{message_text}';
+            message.style.fontSize = '0.9em';
+            message.style.marginTop = '2px';
             
             textContainer.appendChild(heading);
             textContainer.appendChild(message);
@@ -1851,24 +1917,24 @@ def create_interface():
             document.body.appendChild(notification);
             
             // Show notification
-            setTimeout(function() {
+            setTimeout(function() {{
                 notification.style.opacity = '1';
                 notification.style.transform = 'translateX(0)';
                 
                 // Hide after 3 seconds
-                setTimeout(function() {
+                setTimeout(function() {{
                     notification.style.opacity = '0';
                     notification.style.transform = 'translateX(50px)';
                     
                     // Remove element after animation
-                    setTimeout(function() {
+                    setTimeout(function() {{
                         notification.remove();
-                    }, 300);
-                }, 3000);
-            }, 100);
-        })();
+                    }}, 300);
+                }}, 3000);
+            }}, 100);
+        }})();
         </script>
-        <div>Feedback submitted successfully!</div>
+        <div>Feedback {status_text}!</div>
         """
         
         return feedback_response
@@ -1970,7 +2036,7 @@ def create_interface():
                 feedback_stats = gr.HTML(visible=True)
                 
                 # Feedback section
-                with gr.Accordion("Provide Feedback", open=False, elem_id="detector-feedback") as feedback_accordion:
+                with gr.Accordion("Provide Feedback", open=True, elem_id="detector-feedback") as feedback_accordion:
                     gr.Markdown("### Help Improve the System")
                     gr.Markdown("Your feedback helps us refine the hallucination detection system.")
                     
@@ -2113,7 +2179,7 @@ def create_interface():
                         "* <strong style='color: #b2dfdb;'>K</strong>: Weight factor (24 for model pairs)<br>" +
                         "* <strong style='color: #b2dfdb;'>S</strong>: Actual score from user feedback (1 for correct, 0 for incorrect)<br>" +
                         "* <strong style='color: #b2dfdb;'>E</strong>: Expected score based on current rating<br><br>" +
-                        "<em style='color: #80deea;'>E = 1 / (1 + 10<sup>(1500 - ELO_old)/400</sup>)</em></div></div>" +
+                        "<em style='color: #80deea;'>E = 1 / (1 + 10<sup>(1500 - ELO_model)/400</sup>)</em></div></div>" +
                         "<div style='flex: 1; min-width: 280px; padding: 12px; background-color: #455a64; border-radius: 6px; box-shadow: 0 1px 3px rgba(0,0,0,0.12);'>" +
                         "<h4 style='margin-top: 0; color: #ffffff;'>Available Models</h4>" +
                         "<p style='color: #eceff1;'>The system randomly selects from these models for each hallucination detection:</p>" +
@@ -2260,7 +2326,7 @@ def create_interface():
                         "* <strong style='color: #b2dfdb;'>K</strong>: Weight factor (32 for individual models)<br>" +
                         "* <strong style='color: #b2dfdb;'>S</strong>: Actual score (1 for correct judgment, 0 for incorrect)<br>" +
                         "* <strong style='color: #b2dfdb;'>E</strong>: Expected score based on current rating<br><br>" +
-                        "<em style='color: #80deea;'>E = 1 / (1 + 10<sup>(1500 - ELO_old)/400</sup>)</em></div>" +
+                        "<em style='color: #80deea;'>E = 1 / (1 + 10<sup>(1500 - ELO_model)/400</sup>)</em></div>" +
                         "<p style='color: #eceff1; margin-top: 10px;'>All models start with a base ELO of 1500. Scores are updated after each user evaluation.</p></div>" +
                         "<div style='flex: 1; min-width: 280px; padding: 12px; background-color: #455a64; border-radius: 6px; box-shadow: 0 1px 3px rgba(0,0,0,0.12);'>" +
                         "<h4 style='margin-top: 0; color: #ffffff;'>Interpretation Guidelines</h4>" +
@@ -2307,62 +2373,10 @@ def create_interface():
             return ""
         
         # Feedback section is now moved directly inside the Detector tab
-            
-        # Set up interval to update stats
-        with gr.Row(elem_id="stats-container"):
-            with gr.Column():
-                gr.Markdown("### 📊 Live Prediction Accuracy")
-                gr.Markdown("_Auto-refreshes every 5 seconds from MongoDB based on user feedback_")
-                live_stats = gr.HTML(update_stats())
-                
-                # Add loading animation style
-                gr.HTML(
-                    "<style>" +
-                    "@keyframes pulse {" +
-                    "0% { opacity: 0.6; }" +
-                    "50% { opacity: 1; }" +
-                    "100% { opacity: 0.6; }" +
-                    "}" +
-                    ".refreshing::after {" +
-                    "content: \"⟳\";" +
-                    "display: inline-block;" +
-                    "margin-left: 8px;" +
-                    "animation: pulse 1.5s infinite ease-in-out;" +
-                    "color: #2e7d32;" +
-                    "}" +
-                    "#stats-container {" +
-                    "border: 1px solid #b3e5fc;" +
-                    "border-radius: 10px;" +
-                    "padding: 15px;" +
-                    "margin: 10px 0;" +
-                    "background-color: #0277bd;" +
-                    "}" +
-                    "</style>" +
-                    "<div class=\"refreshing\" style=\"text-align: right; font-size: 0.8em; color: #eceff1;\">Auto-refreshing</div>"
-                )
         
-        # Create a refresh button that will be auto-clicked
-        refresh_btn = gr.Button("Refresh Stats", visible=False)
-        refresh_btn.click(
-            fn=update_stats,
-            outputs=[live_stats]
-        )
-        
-        # Add JavaScript to auto-refresh the statistics and enhance the tabs
+        # Add JavaScript to enhance the tabs
         gr.HTML("""
         <script>
-        // Auto-refresh stats every 5 seconds
-        function setupAutoRefresh() {
-            const refreshInterval = 5000; // 5 seconds
-            setInterval(() => {
-                // Find the refresh button by its text and click it
-                const refreshButtons = Array.from(document.querySelectorAll('button'));
-                const refreshBtn = refreshButtons.find(btn => btn.textContent.includes('Refresh Stats'));
-                if (refreshBtn) {
-                    refreshBtn.click();
-                }
-            }, refreshInterval);
-        }
         
         // Add highlighting to the selected tab and handle feedback section visibility
         function setupTabHighlighting() {
@@ -2414,7 +2428,6 @@ def create_interface():
         
         // Set up all JavaScript enhancements after the page loads
         function setupAllEnhancements() {
-            setupAutoRefresh();
             setupTabHighlighting();
             
             // Simple solution to ensure feedback is only visible in detector tab