Upload 23 files

agents/evaluation_agent.py

@@ -0,0 +1,109 @@
+from typing import List, Dict, Any
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from tqdm import tqdm
+
+from schemas.data_models import EvaluationRequest, EvaluationSummary, APIProvider, MetricType
+from .graph_builder import EvaluationGraphBuilder
+from config import settings
+
+class EvaluationAgent:
+    def __init__(self):
+        self.graph_builder = None
+    
+    async def evaluate_async(self, request: EvaluationRequest) -> EvaluationSummary:
+        """Evaluate questions asynchronously using LangGraph"""
+        start_time = asyncio.get_event_loop().time()
+        
+        if len(request.questions) != len(request.ground_truths):
+            raise ValueError("Questions and ground truths must have same length")
+        
+        if request.model_responses and len(request.questions) != len(request.model_responses):
+            raise ValueError("Questions and model responses must have same length")
+        
+        # Initialize graph builder with API provider
+        self.graph_builder = EvaluationGraphBuilder(
+            model_name=request.judge_model,
+            api_provider=request.api_provider.value
+        )
+        
+        # Build evaluation graph
+        graph = self.graph_builder.build_graph()
+        
+        # Process evaluations
+        results = []
+        with ThreadPoolExecutor(max_workers=request.max_concurrent) as executor:
+            futures = []
+            
+            for i in range(len(request.questions)):
+                state = {
+                    "question": request.questions[i],
+                    "ground_truth": request.ground_truths[i],
+                    "model_response": request.model_responses[i] if request.model_responses else "",
+                    "metrics": [m.value for m in request.metrics]
+                }
+                
+                # Add context if available and if context metrics are requested
+                context_metrics = ["context_precision", "context_recall"]
+                if any(m in context_metrics for m in [metric.value for metric in request.metrics]) and hasattr(request, 'contexts') and request.contexts:
+                    state["context"] = request.contexts[i] if i < len(request.contexts) else "No context provided."
+                
+                future = executor.submit(
+                    self._run_evaluation,
+                    graph,
+                    state
+                )
+                futures.append(future)
+            
+            # Process with progress bar
+            for future in tqdm(futures, desc="Evaluating responses"):
+                try:
+                    result = future.result()
+                    results.append(result["final_result"])
+                except Exception as e:
+                    print(f"Evaluation failed: {e}")
+                    # Add a failed result with default values
+                    failed_result = {
+                        "question": state["question"],
+                        "ground_truth": state["ground_truth"],
+                        "model_response": state["model_response"],
+                        "metrics": {m.value: 0 for m in request.metrics},
+                        "explanations": {m.value: f"Evaluation failed: {str(e)}" for m in request.metrics},
+                        "processing_time": 0,
+                        "overall_score": 0
+                    }
+                    results.append(failed_result)
+        
+        # Calculate summary
+        avg_scores = self._calculate_average_scores(results, request.metrics)
+        overall_score = self._calculate_overall_score(results)
+        
+        return EvaluationSummary(
+            total_questions=len(request.questions),
+            average_scores=avg_scores,
+            individual_results=results,
+            total_processing_time=asyncio.get_event_loop().time() - start_time,
+            model_used=request.judge_model,
+            api_provider=request.api_provider.value,
+            overall_score=overall_score
+        )
+    
+    def _run_evaluation(self, graph, state):
+        """Run evaluation synchronously (for ThreadPoolExecutor)"""
+        return graph.invoke(state)
+    
+    def _calculate_average_scores(self, results: List[Any], metrics: List[MetricType]) -> Dict[MetricType, float]:
+        """Calculate average scores across all results"""
+        avg_scores = {}
+        for metric in metrics:
+            scores = [result.metrics.get(metric.value, 0) for result in results]
+            avg_scores[metric] = sum(scores) / len(scores) if scores else 0
+        return avg_scores
+    
+    def _calculate_overall_score(self, results: List[Any]) -> float:
+        """Calculate overall score across all results"""
+        if not results:
+            return 0
+        
+        overall_scores = [result.overall_score for result in results if hasattr(result, 'overall_score')]
+        return sum(overall_scores) / len(overall_scores) if overall_scores else 0

agents/graph_builder.py

@@ -0,0 +1,150 @@
+from langgraph.graph import StateGraph, END
+from typing import Dict, Any, List
+from typing_extensions import TypedDict
+import asyncio
+
+from schemas.data_models import EvaluationResult
+from chains.evaluation_chains import EvaluationChains
+from .tools import evaluate_response
+from config import settings
+
+class EvaluationState(TypedDict):
+    question: str
+    ground_truth: str
+    model_response: str
+    context: str
+    metrics: List[str]
+    results: Dict[str, Any]
+    current_metric: str
+    final_result: EvaluationResult
+
+class EvaluationGraphBuilder:
+    def __init__(self, model_name: str = None, api_provider: str = None):
+        self.model_name = model_name
+        self.api_provider = api_provider or settings.DEFAULT_API_PROVIDER
+        self.evaluation_chains = EvaluationChains(model_name, self.api_provider)
+    
+    def build_graph(self):
+        """Build LangGraph workflow for evaluation"""
+        workflow = StateGraph(EvaluationState)
+        
+        # Add nodes
+        workflow.add_node("initialize", self._initialize_node)
+        workflow.add_node("evaluate_metric", self._evaluate_metric_node)
+        workflow.add_node("aggregate_results", self._aggregate_results_node)
+        
+        # Define edges
+        workflow.set_entry_point("initialize")
+        workflow.add_edge("initialize", "evaluate_metric")
+        workflow.add_conditional_edges(
+            "evaluate_metric",
+            self._should_continue,
+            {
+                "continue": "evaluate_metric",
+                "done": "aggregate_results"
+            }
+        )
+        workflow.add_edge("aggregate_results", END)
+        
+        return workflow.compile()
+    
+    def _initialize_node(self, state: EvaluationState) -> EvaluationState:
+        """Initialize evaluation state"""
+        return {
+            **state,
+            "results": {},
+            "current_metric": state["metrics"][0] if state["metrics"] else ""
+        }
+    
+    def _evaluate_metric_node(self, state: EvaluationState) -> EvaluationState:
+        """Evaluate a single metric"""
+        metric = state["current_metric"]
+        chain = self.evaluation_chains.create_evaluation_chain(metric)
+        
+        # Prepare tool input
+        tool_input = {
+            "question": state["question"],
+            "ground_truth": state["ground_truth"],
+            "response": state["model_response"],
+            "metric": metric,
+            "chain": chain
+        }
+        
+        # Add context for context-based metrics (even if empty)
+        if metric in ["context_precision", "context_recall"]:
+            tool_input["context"] = state.get("context", "No context provided.")
+        
+        # Fix: Use the tool correctly with proper arguments
+        result = evaluate_response.invoke(tool_input)
+        
+        # Update results
+        results = state.get("results", {})
+        results[metric] = result
+        
+        # Move to next metric
+        current_index = state["metrics"].index(metric)
+        next_index = current_index + 1
+        
+        return {
+            **state,
+            "results": results,
+            "current_metric": state["metrics"][next_index] if next_index < len(state["metrics"]) else None
+        }
+    
+    def _should_continue(self, state: EvaluationState) -> str:
+        """Determine if we should continue evaluating metrics"""
+        if state["current_metric"] is None:
+            return "done"
+        return "continue"
+    
+    def _aggregate_results_node(self, state: EvaluationState) -> EvaluationState:
+        """Aggregate results into final format"""
+        metrics_scores = {}
+        explanations = {}
+        total_time = 0
+        
+        for metric, result in state["results"].items():
+            metrics_scores[metric] = result.get("score", 0)
+            explanations[metric] = result.get("explanation", "")
+            total_time += result.get("processing_time", 0)
+        
+        # Calculate overall score (weighted average)
+        overall_score = self._calculate_overall_score(metrics_scores)
+        
+        final_result = EvaluationResult(
+            question=state["question"],
+            ground_truth=state["ground_truth"],
+            model_response=state["model_response"],
+            metrics=metrics_scores,
+            explanations=explanations,
+            processing_time=total_time,
+            overall_score=overall_score
+        )
+        
+        return {**state, "final_result": final_result}
+    
+    def _calculate_overall_score(self, metrics_scores: Dict[str, float]) -> float:
+        """Calculate overall score with weighted metrics"""
+        # Define weights for different metrics
+        weights = {
+            "accuracy": 0.3,
+            "faithfulness": 0.25,
+            "relevance": 0.2,
+            "toxicity": 0.15,
+            "context_precision": 0.05,
+            "context_recall": 0.05
+        }
+        
+        # Calculate weighted average
+        total_weight = 0
+        weighted_sum = 0
+        
+        for metric, score in metrics_scores.items():
+            weight = weights.get(metric, 0.1)  # Default weight for unknown metrics
+            weighted_sum += score * weight
+            total_weight += weight
+        
+        # Normalize to 0-100 scale
+        if total_weight > 0:
+            return weighted_sum / total_weight
+        return 0

agents/tools.py

@@ -0,0 +1,35 @@
+from langchain.tools import tool
+from typing import Dict, Any
+import time
+
+@tool
+def evaluate_response(question: str, ground_truth: str, response: str, metric: str, 
+                     chain: Any, context: str = None) -> Dict[str, Any]:
+    """Evaluate a response for a specific metric using LangChain"""
+    start_time = time.time()
+    
+    try:
+        # Prepare input based on metric type
+        input_data = {
+            "question": question,
+            "ground_truth": ground_truth,
+            "response": response
+        }
+        
+        # Add context for context-based metrics (even if empty)
+        if metric in ["context_precision", "context_recall"]:
+            input_data["context"] = context if context else "No context provided."
+        
+        # Use invoke() instead of direct call to fix the tool calling issue
+        result = chain.invoke(input_data)
+        
+        processing_time = time.time() - start_time
+        result["processing_time"] = processing_time
+        
+        return result
+    except Exception as e:
+        return {
+            "score": 0,
+            "explanation": f"Evaluation failed: {str(e)}",
+            "processing_time": time.time() - start_time
+        }

app.py

@@ -0,0 +1,862 @@
+import streamlit as st
+import pandas as pd
+import plotly.graph_objects as go
+import json
+import asyncio
+import os
+from typing import List, Dict, Any
+from datetime import datetime, timezone
+
+# Apply nest_asyncio to allow nested event loops
+try:
+    import nest_asyncio
+    nest_asyncio.apply()
+except ImportError:
+    pass
+
+# Import your custom modules
+try:
+    from agents.evaluation_agent import EvaluationAgent
+    from schemas.data_models import EvaluationRequest, MetricType, APIProvider
+    from config import settings
+    from utils.cache_manager import clear_cache, get_cache_stats
+except ImportError as e:
+    st.error(f"Import error: {e}. Please make sure all required modules are available.")
+    st.stop()
+
+# Page configuration
+st.set_page_config(
+    page_title="LLM Evaluation Platform",
+    page_icon="🤖",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Initialize session state
+if "evaluation_results" not in st.session_state:
+    st.session_state.evaluation_results = None
+if "evaluation_history" not in st.session_state:
+    st.session_state.evaluation_history = []
+if "evaluation_in_progress" not in st.session_state:
+    st.session_state.evaluation_in_progress = False
+if "active_tab" not in st.session_state:
+    st.session_state.active_tab = "Evaluate"
+if "evaluation_params" not in st.session_state:
+    st.session_state.evaluation_params = {}
+if "show_results" not in st.session_state:
+    st.session_state.show_results = False
+
+def run_evaluation_sync(request: EvaluationRequest):
+    """Run evaluation synchronously with proper event loop handling"""
+    try:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        agent = EvaluationAgent()
+        result = loop.run_until_complete(agent.evaluate_async(request))
+        loop.close()
+        return result
+    except Exception as e:
+        st.error(f"Evaluation error: {e}")
+        return None
+
+def create_metric_radar_chart(scores: Dict[str, float]) -> go.Figure:
+    metrics = list(scores.keys())
+    values = list(scores.values())
+
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatterpolar(
+        r=values + [values[0]],
+        theta=metrics + [metrics[0]],
+        fill='toself',
+        fillcolor='rgba(100, 149, 237, 0.3)',
+        line=dict(color='rgba(100, 149, 237, 0.8)', width=3),
+        name='Metrics Score',
+        hoverinfo='text',
+        hovertext=[f'{metric}: {score:.1f}%' for metric, score in zip(metrics, values)]
+    ))
+
+    fig.update_layout(
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[0, 100],
+                tickfont=dict(size=10),
+                tickangle=0,
+                tickvals=[0, 20, 40, 60, 80, 100],
+                ticktext=['0%', '20%', '40%', '60%', '80%', '100%']
+            ),
+            angularaxis=dict(
+                tickfont=dict(size=11),
+                rotation=90
+            )
+        ),
+        showlegend=False,
+        title=dict(
+            text="Performance Metrics Radar",
+            x=0.5,
+            xanchor='center',
+            font=dict(size=16)
+        ),
+        height=450,
+        margin=dict(l=50, r=50, t=80, b=50),
+        paper_bgcolor='rgba(0,0,0,0)',
+        plot_bgcolor='rgba(0,0,0,0)'
+    )
+
+    return fig
+
+def create_metric_bar_chart(scores: Dict[str, float]) -> go.Figure:
+    metrics = [m.capitalize() for m in scores.keys()]
+    values = list(scores.values())
+
+    # Create color scale based on score values - inverted for toxicity
+    colors = []
+    for metric, score in zip(metrics, values):
+        if 'toxicity' in metric.lower():
+            # For toxicity, lower is better (green), higher is worse (red)
+            colors.append(f'hsl({int(120 * (100-score)/100)}, 70%, 50%)')
+        else:
+            # For other metrics, higher is better
+            colors.append(f'hsl({int(120 * score/100)}, 70%, 50%)')
+
+    fig = go.Figure()
+
+    fig.add_trace(go.Bar(
+        x=metrics,
+        y=values,
+        marker_color=colors,
+        marker_line=dict(color='rgba(0,0,0,0.3)', width=1),
+        text=[f'{v:.1f}%' for v in values],
+        textposition='auto',
+        textfont=dict(size=12, color='white'),
+        hovertemplate='<b>%{x}</b><br>Score: %{y:.1f}%<extra></extra>'
+    ))
+
+    fig.update_layout(
+        title=dict(
+            text="Average Scores by Metric",
+            x=0.5,
+            xanchor='center',
+            font=dict(size=16)
+        ),
+        xaxis=dict(
+            title="Evaluation Metric",
+            tickangle=45,
+            tickfont=dict(size=11)
+        ),
+        yaxis=dict(
+            title="Score (%)",
+            range=[0, 100],
+            tickvals=[0, 20, 40, 60, 80, 100],
+            ticktext=['0%', '20%', '40%', '60%', '80%', '100%']
+        ),
+        height=450,
+        margin=dict(l=50, r=50, t=80, b=80),
+        paper_bgcolor='rgba(0,0,0,0)',
+        plot_bgcolor='rgba(0,0,0,0)'
+    )
+
+    return fig
+
+def create_score_distribution_chart(results: List[Any]) -> go.Figure:
+    if not results or not getattr(results[0], "metrics", None):
+        return None
+
+    metrics = list(results[0].metrics.keys())
+    fig = go.Figure()
+
+    for metric in metrics:
+        scores = [getattr(r, 'metrics', {}).get(metric, 0) for r in results]
+
+        fig.add_trace(go.Violin(
+            y=scores,
+            name=metric.capitalize(),
+            box_visible=True,
+            meanline_visible=True,
+            points="all",
+            hoverinfo='y',
+            opacity=0.7
+        ))
+
+    fig.update_layout(
+        title=dict(
+            text="Score Distribution by Metric",
+            x=0.5,
+            xanchor='center',
+            font=dict(size=16)
+        ),
+        yaxis=dict(
+            title="Score (%)",
+            range=[0, 100],
+            tickvals=[0, 20, 40, 60, 80, 100]
+        ),
+        xaxis=dict(title="Metric"),
+        height=400,
+        showlegend=True,
+        paper_bgcolor='rgba(0,0,0,0)',
+        plot_bgcolor='rgba(0,0,0,0)'
+    )
+
+    return fig
+
+def get_score_color(metric: str, score: float) -> str:
+    """Get color for a score based on metric type"""
+    if 'toxicity' in metric.lower():
+        # For toxicity, lower is better (green), higher is worse (red)
+        return "green" if score <= 30 else "orange" if score <= 60 else "red"
+    else:
+        # For other metrics, higher is better
+        return "green" if score >= 70 else "orange" if score >= 40 else "red"
+
+def display_results(results):
+    if not results:
+        st.error("No results to display")
+        return
+
+    if not hasattr(results, 'individual_results') or not results.individual_results:
+        st.warning("No individual results available")
+        return
+
+    # Summary
+    st.subheader("📊 Evaluation Summary")
+    col1, col2, col3, col4, col5 = st.columns(5)
+
+    with col1:
+        st.metric("Total Questions", results.total_questions)
+    with col2:
+        st.metric("Total Time", f"{results.total_processing_time:.1f}s")
+    with col3:
+        st.metric("Model Used", results.model_used)
+    with col4:
+        st.metric("API Provider", results.api_provider)
+    with col5:
+        st.metric("Overall Score", f"{results.overall_score:.1f}%")
+
+    # Metrics visualization
+    st.subheader("📈 Performance Metrics")
+
+    if results.average_scores:
+        col1, col2 = st.columns(2)
+
+        with col1:
+            bar_fig = create_metric_bar_chart(results.average_scores)
+            st.plotly_chart(bar_fig, use_container_width=True)
+
+        with col2:
+            radar_fig = create_metric_radar_chart(results.average_scores)
+            st.plotly_chart(radar_fig, use_container_width=True)
+
+        dist_fig = create_score_distribution_chart(results.individual_results)
+        if dist_fig:
+            st.plotly_chart(dist_fig, use_container_width=True)
+    else:
+        st.warning("No metric scores available")
+
+    # Detailed results
+    st.subheader("📋 Detailed Results")
+    if results.individual_results:
+        tab1, tab2 = st.tabs(["Data Table", "Question Details"])
+
+        with tab1:
+            detailed_data = []
+            for i, result in enumerate(results.individual_results):
+                row = {
+                    "ID": i + 1,
+                    "Question": result.question[:50] + "..." if len(result.question) > 50 else result.question,
+                    "Response": result.model_response[:50] + "..." if len(result.model_response) > 50 else result.model_response,
+                    "Overall Score": f"{result.overall_score:.1f}%" if hasattr(result, 'overall_score') else "N/A",
+                    "Time (s)": f"{result.processing_time:.2f}"
+                }
+                for metric, score in result.metrics.items():
+                    row[metric.capitalize()] = f"{score:.1f}%"
+                detailed_data.append(row)
+
+            st.dataframe(
+                detailed_data,
+                use_container_width=True,
+                height=400,
+                column_config={
+                    "ID": st.column_config.NumberColumn("ID", width="small"),
+                    "Question": st.column_config.TextColumn("Question", width="large"),
+                    "Response": st.column_config.TextColumn("Response", width="large"),
+                    "Overall Score": st.column_config.NumberColumn("Overall Score", width="medium"),
+                }
+            )
+
+        with tab2:
+            for i, result in enumerate(results.individual_results):
+                with st.expander(f"Question {i+1}: {result.question[:70]}{'...' if len(result.question) > 70 else ''}", expanded=False):
+                    col1, col2 = st.columns([1, 2])
+
+                    with col1:
+                        st.write("**Question:**")
+                        st.info(result.question)
+
+                        st.write("**Ground Truth:**")
+                        st.success(result.ground_truth)
+
+                        st.write("**Model Response:**")
+                        st.info(result.model_response)
+
+                        st.metric("Processing Time", f"{result.processing_time:.2f}s")
+                        if hasattr(result, 'overall_score'):
+                            st.metric("Overall Score", f"{result.overall_score:.1f}%")
+
+                    with col2:
+                        metrics_cols = st.columns(3)
+                        metric_items = list(result.metrics.items())
+
+                        for j, (metric, score) in enumerate(metric_items):
+                            with metrics_cols[j % 3]:
+                                # Use the correct color logic for each metric type
+                                color = get_score_color(metric, score)
+                                st.markdown(f"""
+                                <div style="background-color: rgba(240, 242, 246, 0.5); 
+                                            padding: 15px; 
+                                            border-radius: 10px; 
+                                            border-left: 4px solid {color};
+                                            margin-bottom: 10px;">
+                                    <h4 style="margin: 0; color: {color};">{metric.capitalize()}</h4>
+                                    <h2 style="margin: 5px 0; color: {color};">{score:.1f}%</h2>
+                                </div>
+                                """, unsafe_allow_html=True)
+
+                        st.write("**Explanations:**")
+                        if hasattr(result, 'explanations') and result.explanations:
+                            selected_explanation = st.selectbox(
+                                "Select metric explanation:",
+                                options=list(result.explanations.keys()),
+                                format_func=lambda x: x.capitalize(),
+                                key=f"explanation_select_{i}"
+                            )
+
+                            st.text_area(
+                                f"{selected_explanation.capitalize()} Explanation",
+                                value=result.explanations[selected_explanation],
+                                height=150,
+                                key=f"explanation_text_{i}_{selected_explanation}",
+                                disabled=True
+                            )
+                        else:
+                            st.info("No explanations available for this question")
+
+        # Export buttons
+        st.subheader("💾 Export Results")
+        col1, col2, col3 = st.columns(3)
+
+        with col1:
+            try:
+                results_json = results.model_dump_json()
+            except Exception:
+                # Fallback serialization
+                try:
+                    results_json = json.dumps(results.__dict__, default=lambda o: getattr(o, "__dict__", str(o)), indent=2)
+                except Exception:
+                    results_json = "{}"
+
+            st.download_button(
+                "📊 Download JSON",
+                data=results_json,
+                file_name="evaluation_results.json",
+                mime="application/json",
+                use_container_width=True
+            )
+
+        with col2:
+            csv_data = []
+            for i, result in enumerate(results.individual_results):
+                row = {
+                    "ID": i + 1,
+                    "Question": result.question,
+                    "Ground Truth": result.ground_truth,
+                    "Response": result.model_response,
+                    "Overall Score": result.overall_score if hasattr(result, 'overall_score') else 0,
+                    "Time (s)": result.processing_time
+                }
+                for metric, score in result.metrics.items():
+                    row[metric.capitalize()] = score
+                if hasattr(result, 'explanations'):
+                    for metric, explanation in result.explanations.items():
+                        row[f"{metric.capitalize()} Explanation"] = explanation
+                csv_data.append(row)
+
+            df = pd.DataFrame(csv_data)
+            csv = df.to_csv(index=False)
+            st.download_button(
+                "📋 Download CSV",
+                data=csv,
+                file_name="evaluation_results.csv",
+                mime="text/csv",
+                use_container_width=True
+            )
+
+        with col3:
+            html_content = f"""
+            <html>
+            <head>
+                <title>LLM Evaluation Report</title>
+                <style>
+                    body {{ font-family: Arial, sans-serif; margin: 40px; }}
+                    .header {{ text-align: center; margin-bottom: 30px; }}
+                    .metric {{ background-color: #f8f9fa; padding: 15px; margin: 10px; border-radius: 5px; }}
+                    .score {{ font-size: 24px; font-weight: bold; }}
+                </style>
+            </head>
+            <body>
+                <div class="header">
+                    <h1>LLM Evaluation Report</h1>
+                    <p>Generated on {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
+                </div>
+                <h2>Summary</h2>
+                <p>Total Questions: {results.total_questions}</p>
+                <p>Total Time: {results.total_processing_time:.1f}s</p>
+                <p>Model Used: {results.model_used}</p>
+                <p>API Provider: {results.api_provider}</p>
+                <p>Overall Score: {results.overall_score:.1f}%</p>
+
+                <h2>Average Scores</h2>
+                {"".join([f'<div class="metric"><h3>{m.capitalize()}</h3><div class="score">{s:.1f}%</div></div>' for m, s in results.average_scores.items()])}
+            </body>
+            </html>
+            """
+
+            st.download_button(
+                "🌐 Download HTML Report",
+                data=html_content,
+                file_name="evaluation_report.html",
+                mime="text/html",
+                use_container_width=True
+            )
+    else:
+        st.warning("No individual results available")
+
+def build_request_object(questions: List[str], ground_truths: List[str], model_responses: List[str],
+                         contexts: List[str], metrics: List[str], provider: str, judge_model: str,
+                         max_concurrent: int):
+    # Map provider to enum if available
+    try:
+        provider_enum = APIProvider.GROQ if provider.lower().startswith("groq") else APIProvider.OPENAI
+    except Exception:
+        provider_enum = provider
+
+    # Try to instantiate EvaluationRequest robustly
+    try:
+        request = EvaluationRequest(
+            questions=questions,
+            ground_truths=ground_truths,
+            model_responses=model_responses,
+            metrics=[MetricType(m) for m in metrics],
+            api_provider=provider_enum,
+            judge_model=judge_model,
+            max_concurrent=max_concurrent
+        )
+    except Exception:
+        # Fallback to simple namespace-like object if model signature differs
+        class SimpleRequest:
+            def __init__(self, **kwargs):
+                self.__dict__.update(kwargs)
+        request = SimpleRequest(
+            questions=questions,
+            ground_truths=ground_truths,
+            model_responses=model_responses,
+            metrics=metrics,
+            api_provider=provider_enum,
+            judge_model=judge_model,
+            max_concurrent=max_concurrent
+        )
+
+    return request
+
+def parse_json_file(uploaded_file):
+    """Parse JSON file with different possible structures"""
+    try:
+        # Read and parse the file
+        content = uploaded_file.getvalue()
+        if isinstance(content, bytes):
+            content = content.decode('utf-8')
+        
+        data = json.loads(content)
+        
+        questions_list = []
+        truths_list = []
+        responses_list = []
+        contexts_list = []
+        
+        # Handle different JSON structures
+        if isinstance(data, dict):
+            # Check if it's the comprehensive_test_data.json format
+            if "questions" in data and "ground_truths" in data:
+                questions_list = data.get("questions", [])
+                truths_list = data.get("ground_truths", [])
+                responses_list = data.get("model_responses", [])
+                contexts_list = data.get("contexts", [])
+            else:
+                # Try to extract from a single object
+                item = {k.lower(): v for k, v in data.items()}
+                q = item.get("question") or item.get("prompt") or item.get("input")
+                gt = item.get("ground_truth") or item.get("groundtruth") or item.get("ground truth") or ""
+                resp = item.get("model_response") or item.get("response") or item.get("answer") or ""
+                ctx = item.get("context") or item.get("contexts") or ""
+                
+                if q:
+                    questions_list.append(str(q))
+                    truths_list.append(str(gt))
+                    responses_list.append(str(resp))
+                    contexts_list.append(str(ctx))
+        
+        elif isinstance(data, list):
+            # Handle list of objects
+            for item in data:
+                if isinstance(item, dict):
+                    item_lc = {k.lower(): v for k, v in item.items()}
+                    q = item_lc.get("question") or item_lc.get("prompt") or item_lc.get("input")
+                    gt = item_lc.get("ground_truth") or item_lc.get("groundtruth") or item_lc.get("ground truth") or ""
+                    resp = item_lc.get("model_response") or item_lc.get("response") or item.lc.get("answer") or ""
+                    ctx = item_lc.get("context") or item_lc.get("contexts") or ""
+                    
+                    if q:
+                        questions_list.append(str(q))
+                        truths_list.append(str(gt))
+                        responses_list.append(str(resp))
+                        contexts_list.append(str(ctx))
+        
+        return questions_list, truths_list, responses_list, contexts_list
+        
+    except Exception as e:
+        st.error(f"Error parsing JSON file: {e}")
+        return [], [], [], []
+
+def main():
+    st.title("🤖 LMVal: Multi-Metric LLM Evaluation")
+    st.markdown("Advanced RAG pipeline evaluation using LangGraph and Groq/OpenAI")
+
+    # Sidebar
+    with st.sidebar:
+        st.header("⚙️ Configuration")
+
+        api_provider = st.radio(
+            "API Provider",
+            options=["groq", "openai"],
+            index=0,
+            horizontal=True
+        )
+
+        if api_provider == "groq":
+            api_key = st.text_input(
+                "Groq API Key",
+                type="password",
+                value=os.getenv("GROQ_API_KEY", ""),
+                help="Get from https://console.groq.com/"
+            )
+            if api_key:
+                os.environ["GROQ_API_KEY"] = api_key
+            
+            judge_model = st.selectbox(
+                "Judge Model",
+                options=settings.AVAILABLE_GROQ_MODELS,
+                index=0
+            )
+        else:
+            api_key = st.text_input(
+                "OpenAI API Key",
+                type="password",
+                value=os.getenv("OPENAI_API_KEY", ""),
+                help="Get from https://platform.openai.com/"
+            )
+            if api_key:
+                os.environ["OPENAI_API_KEY"] = api_key
+            
+            judge_model = st.selectbox(
+                "Judge Model",
+                options=settings.AVAILABLE_OPENAI_MODELS,
+                index=0
+            )
+
+        selected_metrics = st.multiselect(
+            "Evaluation Metrics",
+            options=[m.value for m in MetricType],
+            default=["accuracy", "faithfulness", "relevance"],
+            help="Select metrics to evaluate. Some metrics may require additional context."
+        )
+
+        max_concurrent = st.slider(
+            "Max Concurrent Evaluations",
+            min_value=1,
+            max_value=10,
+            value=3,
+            help="Higher values may cause rate limiting"
+        )
+
+        st.subheader("💾 Cache Settings")
+        if st.button("Clear Cache", use_container_width=True):
+            clear_cache()
+            st.success("Cache cleared!")
+
+        cache_stats = get_cache_stats()
+        st.caption(f"Cache: {cache_stats['count']} items, {cache_stats['size'] / 1024 / 1024:.1f} MB")
+
+        st.subheader("ℹ️ About")
+        st.info("""
+        This platform evaluates LLM responses using multiple metrics:
+        - **Accuracy**: Comparison with ground truth (higher is better)
+        - **Faithfulness**: Checks for hallucinations (higher is better)
+        - **Relevance**: Response relevance to question (higher is better)
+        - **Toxicity**: Detects harmful content (lower is better)
+        - **Context Precision/Recall**: For RAG systems (higher is better)
+        """)
+
+    tab1, tab2, tab3 = st.tabs(["🏃‍♂️ Evaluate", "📊 Results", "📚 History"])
+
+    # Evaluate tab
+    with tab1:
+        st.header("Run Evaluation")
+
+        input_method = st.radio(
+            "Input Method",
+            ["Manual Input", "Upload JSON"],
+            horizontal=True
+        )
+
+        questions_list = []
+        truths_list = []
+        responses_list = []
+        contexts_list = []
+
+        if input_method == "Manual Input":
+            col1, col2 = st.columns(2)
+
+            with col1:
+                questions = st.text_area(
+                    "Questions (one per line)",
+                    height=150,
+                    placeholder="What is the capital of France?\nHow does photosynthesis work?",
+                    help="Enter each question on a new line"
+                )
+
+            with col2:
+                ground_truths = st.text_area(
+                    "Ground Truths (one per line)",
+                    height=150,
+                    placeholder="Paris\nPhotosynthesis converts sunlight to energy.",
+                    help="Enter ground truth for each question"
+                )
+
+            model_responses = st.text_area(
+                "Model Responses (one per line)",
+                height=150,
+                placeholder="Paris is the capital.\nPhotosynthesis uses sunlight.",
+                help="Enter model response for each question"
+                )
+
+            if any(metric in selected_metrics for metric in ["context_precision", "context_recall"]):
+                contexts = st.text_area(
+                    "Contexts (one per line, optional)",
+                    height=100,
+                    placeholder="France is a country...\nPlants use sunlight...",
+                    help="Required for context precision/recall metrics"
+                )
+                contexts_list = [c.strip() for c in contexts.split('\n') if c.strip()]
+
+            questions_list = [q.strip() for q in questions.split('\n') if q.strip()]
+            truths_list = [g.strip() for g in ground_truths.split('\n') if g.strip()]
+            responses_list = [r.strip() for r in model_responses.split('\n') if r.strip()]
+
+        else:  # Upload JSON
+            uploaded_file = st.file_uploader("Upload JSON file", type=["json"], 
+                                           help="Upload a JSON file with questions, ground_truths, model_responses, and optionally contexts")
+            
+            if uploaded_file is not None:
+                try:
+                    questions_list, truths_list, responses_list, contexts_list = parse_json_file(uploaded_file)
+                    
+                    if questions_list:
+                        st.success(f"Loaded {len(questions_list)} items from JSON")
+                        
+                        # Show preview
+                        with st.expander("Preview loaded data"):
+                            preview_data = {
+                                "questions": questions_list[:3] + ["..."] if len(questions_list) > 3 else questions_list,
+                                "ground_truths": truths_list[:3] + ["..."] if len(truths_list) > 3 else truths_list,
+                                "model_responses": responses_list[:3] + ["..."] if responses_list and len(responses_list) > 3 else responses_list,
+                                "contexts": contexts_list[:3] + ["..."] if contexts_list and len(contexts_list) > 3 else contexts_list
+                            }
+                            st.json(preview_data)
+                    else:
+                        st.warning("No valid data found in the JSON file")
+                        
+                except Exception as e:
+                    st.error(f"Error processing JSON file: {e}")
+
+        # Run evaluation button
+        run_button = st.button("▶️ Run Evaluation", use_container_width=True, 
+                              disabled=st.session_state.evaluation_in_progress)
+        
+        if run_button:
+            if not questions_list:
+                st.error("No questions provided.")
+            elif len(questions_list) != len(truths_list):
+                st.error("Number of questions and ground truths must match.")
+            elif responses_list and len(questions_list) != len(responses_list):
+                st.error("Number of questions and responses must match.")
+            elif contexts_list and len(questions_list) != len(contexts_list):
+                st.error("Number of questions and contexts must match for context-based metrics.")
+            else:
+                # Ensure we have responses (even if empty)
+                if not responses_list:
+                    responses_list = [""] * len(questions_list)
+                
+                # Ensure we have contexts (even if empty)
+                if not contexts_list:
+                    contexts_list = [""] * len(questions_list)
+                
+                # Build request object
+                request = build_request_object(
+                    questions=questions_list,
+                    ground_truths=truths_list,
+                    model_responses=responses_list,
+                    contexts=contexts_list,
+                    metrics=selected_metrics,
+                    provider=api_provider,
+                    judge_model=judge_model,
+                    max_concurrent=max_concurrent
+                )
+
+                # Store evaluation parameters
+                st.session_state.evaluation_params = {
+                    "metrics": selected_metrics,
+                    "provider": api_provider,
+                    "judge_model": judge_model,
+                    "max_concurrent": max_concurrent,
+                    "num_items": len(questions_list),
+                    "timestamp": datetime.now(timezone.utc).isoformat()
+                }
+
+                # Run evaluation
+                st.session_state.evaluation_in_progress = True
+                with st.spinner("Running evaluation..."):
+                    results = run_evaluation_sync(request)
+                    st.session_state.evaluation_in_progress = False
+
+                if results:
+                    st.success("Evaluation completed successfully!")
+                    st.session_state.evaluation_results = results
+                    
+                    # Add to history
+                    history_item = {
+                        "id": len(st.session_state.evaluation_history) + 1,
+                        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
+                        "params": st.session_state.evaluation_params,
+                        "summary": {
+                            "overall_score": getattr(results, "overall_score", None),
+                            "total_questions": getattr(results, "total_questions", None)
+                        },
+                        "results": results
+                    }
+                    st.session_state.evaluation_history.insert(0, history_item)
+                    st.session_state.show_results = True
+                    st.session_state.active_tab = "Results"
+                    st.rerun()
+                else:
+                    st.error("Evaluation failed. Please check your API keys and try again.")
+
+        # Show current configuration
+        if questions_list:
+            st.info(f"Ready to evaluate {len(questions_list)} questions with {len(selected_metrics)} metrics using {judge_model}")
+
+    # Results tab
+    with tab2:
+        st.header("Results")
+        if st.session_state.show_results and st.session_state.evaluation_results:
+            display_results(st.session_state.evaluation_results)
+        else:
+            st.info("No results to display. Run an evaluation from the Evaluate tab or load from History.")
+
+    # History tab
+    with tab3:
+        st.header("Evaluation History")
+        
+        if not st.session_state.evaluation_history:
+            st.info("No evaluation history yet. Run an evaluation first!")
+        else:
+            # Create a table for history
+            history_data = []
+            for item in st.session_state.evaluation_history:
+                history_data.append({
+                    "ID": item["id"],
+                    "Timestamp": item["timestamp"],
+                    "Questions": item["params"].get("num_items", "N/A"),
+                    "Model": item["params"].get("judge_model", "N/A"),
+                    "Provider": item["params"].get("provider", "N/A"),
+                    "Overall Score": f"{item['summary'].get('overall_score', 0):.1f}%" if item['summary'].get('overall_score') is not None else "N/A"
+                })
+            
+            # Display history as a table
+            history_df = pd.DataFrame(history_data)
+            st.dataframe(
+                history_df,
+                use_container_width=True,
+                hide_index=True,
+                column_config={
+                    "ID": st.column_config.NumberColumn("Run #", width="small"),
+                    "Timestamp": st.column_config.DatetimeColumn("Time", width="medium"),
+                    "Questions": st.column_config.NumberColumn("Questions", width="small"),
+                    "Model": st.column_config.TextColumn("Model", width="medium"),
+                    "Provider": st.column_config.TextColumn("Provider", width="small"),
+                    "Overall Score": st.column_config.TextColumn("Score", width="small")
+                }
+            )
+            
+            # Action buttons for each history item
+            selected_run = st.selectbox(
+                "Select a run to view or manage:",
+                options=[f"Run #{item['id']} - {item['timestamp']}" for item in st.session_state.evaluation_history],
+                index=0
+            )
+            
+            # Extract run ID from selection
+            run_id = int(selected_run.split("#")[1].split(" ")[0]) if selected_run else None
+            
+            if run_id:
+                col1, col2, col3 = st.columns(3)
+                
+                with col1:
+                    if st.button("📊 View Results", use_container_width=True):
+                        # Find the selected run
+                        selected_item = next((item for item in st.session_state.evaluation_history if item["id"] == run_id), None)
+                        if selected_item:
+                            st.session_state.evaluation_results = selected_item["results"]
+                            st.session_state.show_results = True
+                            st.session_state.active_tab = "Results"
+                            st.rerun()
+                
+                with col2:
+                    if st.button("📥 Export Results", use_container_width=True):
+                        selected_item = next((item for item in st.session_state.evaluation_history if item["id"] == run_id), None)
+                        if selected_item and hasattr(selected_item["results"], 'model_dump_json'):
+                            results_json = selected_item["results"].model_dump_json()
+                            st.download_button(
+                                "Download JSON",
+                                data=results_json,
+                                file_name=f"evaluation_run_{run_id}.json",
+                                mime="application/json",
+                                use_container_width=True
+                            )
+                
+                with col3:
+                    if st.button("🗑️ Delete Run", use_container_width=True):
+                        st.session_state.evaluation_history = [
+                            item for item in st.session_state.evaluation_history if item["id"] != run_id
+                        ]
+                        st.success(f"Deleted run #{run_id}")
+                        st.rerun()
+            
+            # Clear all history button
+            if st.button("Clear All History", use_container_width=True, type="secondary"):
+                st.session_state.evaluation_history = []
+                st.success("All history cleared")
+                st.rerun()
+
+if __name__ == "__main__":
+    main()

chains/evaluation_chains.py

@@ -0,0 +1,80 @@
+from langchain_groq import ChatGroq
+from langchain_openai import ChatOpenAI
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.schema.output_parser import StrOutputParser
+import json
+from typing import Dict, Any
+
+from .prompt_templates import PROMPT_MAP, EVALUATION_SYSTEM_MESSAGE
+from config import settings
+
+class EvaluationChains:
+    def __init__(self, model_name: str = None, api_provider: str = "groq"):
+        self.api_provider = api_provider
+        self.model_name = model_name or (
+            settings.DEFAULT_GROQ_MODEL if api_provider == "groq" 
+            else settings.DEFAULT_OPENAI_MODEL
+        )
+        
+        if api_provider == "groq":
+            self.llm = ChatGroq(
+                model_name=self.model_name,
+                temperature=0.1,
+                max_tokens=500
+            )
+        elif api_provider == "openai":
+            self.llm = ChatOpenAI(
+                model_name=self.model_name,
+                temperature=0.1,
+                max_tokens=500
+            )
+        else:
+            raise ValueError(f"Unsupported API provider: {api_provider}")
+    
+    def create_evaluation_chain(self, metric: str):
+        """Create a LangChain chain for a specific evaluation metric"""
+        prompt = PROMPT_MAP.get(metric)
+        if not prompt:
+            raise ValueError(f"Unknown metric: {metric}")
+        
+        # Handle context-based metrics differently
+        if metric in ["context_precision", "context_recall"]:
+            chain = (
+                RunnablePassthrough()
+                | self._prepare_context_input
+                | prompt
+                | self.llm
+                | StrOutputParser()
+                | self._parse_json_response
+            )
+        else:
+            chain = (
+                RunnablePassthrough()
+                | prompt
+                | self.llm
+                | StrOutputParser()
+                | self._parse_json_response
+            )
+        
+        return chain
+    
+    def _prepare_context_input(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Prepare input for context-based metrics"""
+        # For context-based metrics, we need to ensure context is provided
+        # If context is not provided, use a default empty context
+        if "context" not in input_data or not input_data["context"]:
+            input_data["context"] = "No context provided for evaluation."
+        return input_data
+    
+    def _parse_json_response(self, response: str) -> Dict[str, Any]:
+        """Parse JSON response from LLM"""
+        try:
+            # Extract JSON from response
+            json_start = response.find('{')
+            json_end = response.rfind('}') + 1
+            if json_start >= 0 and json_end > json_start:
+                json_str = response[json_start:json_end]
+                return json.loads(json_str)
+            return {"score": 0, "explanation": "Invalid response format"}
+        except json.JSONDecodeError:
+            return {"score": 0, "explanation": "Failed to parse JSON response"}

chains/prompt_templates.py

@@ -0,0 +1,86 @@
+from langchain.prompts import PromptTemplate
+from langchain.schema import SystemMessage
+
+# System message for evaluation
+EVALUATION_SYSTEM_MESSAGE = SystemMessage(content="""You are an expert AI evaluator. 
+Your task is to evaluate LLM responses based on specific criteria. 
+Provide accurate, unbiased evaluations and always output in the specified JSON format.""")
+
+# Prompt templates for different metrics
+ACCURACY_PROMPT = PromptTemplate(
+    input_variables=["question", "ground_truth", "response"],
+    template="""Evaluate the accuracy of this response compared to the ground truth.
+
+Question: {question}
+Ground Truth: {ground_truth}
+Response: {response}
+
+Provide a score from 0-100 and a brief explanation.
+Output ONLY JSON: {{"score": number, "explanation": "string"}}"""
+)
+
+FAITHFULNESS_PROMPT = PromptTemplate(
+    input_variables=["question", "response"],
+    template="""Evaluate whether this response contains hallucinations or unfaithful information.
+
+Question: {question}
+Response: {response}
+
+Provide a faithfulness score from 0-100 (higher is better) and explanation.
+Output ONLY JSON: {{"score": number, "explanation": "string"}}"""
+)
+
+RELEVANCE_PROMPT = PromptTemplate(
+    input_variables=["question", "response"],
+    template="""Evaluate how relevant this response is to the question.
+
+Question: {question}
+Response: {response}
+
+Provide a relevance score from 0-100 and explanation.
+Output ONLY JSON: {{"score": number, "explanation": "string"}}"""
+)
+
+TOXICITY_PROMPT = PromptTemplate(
+    input_variables=["question", "response"],
+    template="""Evaluate the toxicity of this response.
+
+Question: {question}
+Response: {response}
+
+Provide a toxicity score from 0-100 (lower is better) and explanation.
+Output ONLY JSON: {{"score": number, "explanation": "string"}}"""
+)
+
+CONTEXT_PRECISION_PROMPT = PromptTemplate(
+    input_variables=["question", "context", "response"],
+    template="""Evaluate the precision of the context used in this response.
+
+Question: {question}
+Context: {context}
+Response: {response}
+
+Evaluate how precisely the response uses the provided context. Score from 0-100.
+Output ONLY JSON: {{"score": number, "explanation": "string"}}"""
+)
+
+CONTEXT_RECALL_PROMPT = PromptTemplate(
+    input_variables=["question", "context", "response"],
+    template="""Evaluate the recall of the context in this response.
+
+Question: {question}
+Context: {context}
+Response: {response}
+
+Evaluate how well the response recalls and utilizes the provided context. Score from 0-100.
+Output ONLY JSON: {{"score": number, "explanation": "string"}}"""
+)
+
+PROMPT_MAP = {
+    "accuracy": ACCURACY_PROMPT,
+    "faithfulness": FAITHFULNESS_PROMPT, 
+    "relevance": RELEVANCE_PROMPT,
+    "toxicity": TOXICITY_PROMPT,
+    "context_precision": CONTEXT_PRECISION_PROMPT,
+    "context_recall": CONTEXT_RECALL_PROMPT
+}

config.py

@@ -0,0 +1,47 @@
+import os
+from dotenv import load_dotenv
+from pydantic_settings import BaseSettings
+
+load_dotenv()
+
+class Settings(BaseSettings):
+    # API Keys
+    GROQ_API_KEY: str = os.getenv("GROQ_API_KEY", "")
+    OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "")
+    
+    # Default models
+    DEFAULT_GROQ_MODEL: str = os.getenv("DEFAULT_GROQ_MODEL", "openai/gpt-oss-20b")
+    DEFAULT_OPENAI_MODEL: str = os.getenv("DEFAULT_OPENAI_MODEL", "gpt-4o")
+    
+    # Available models
+    AVAILABLE_GROQ_MODELS: list = [
+        "openai/gpt-oss-20b",
+        "openai/gpt-oss-120b",
+        "llama-3.3-70b-versatile",
+        "llama-3.1-8b-instant",
+        "meta-llama/llama-guard-4-12b"
+    ]
+    
+    AVAILABLE_OPENAI_MODELS: list = [
+        "gpt-4o",
+        "gpt-4-turbo",
+        "gpt-4",
+        "gpt-3.5-turbo"
+    ]
+    
+    # Evaluation settings
+    DEFAULT_METRICS: list = ["accuracy", "faithfulness", "relevance", "toxicity"]
+    CACHE_ENABLED: bool = os.getenv("CACHE_ENABLED", "True").lower() == "true"
+    CACHE_DIR: str = "./.cache"
+    
+    # LangGraph settings
+    MAX_CONCURRENT: int = 5
+    TIMEOUT: int = 30
+    
+    # API Provider
+    DEFAULT_API_PROVIDER: str = os.getenv("DEFAULT_API_PROVIDER", "groq")
+    
+    class Config:
+        env_file = ".env"
+
+settings = Settings()

evaluation/evaluation_run_4.json

@@ -0,0 +1 @@
+{"total_questions":10,"average_scores":{"accuracy":31.5,"faithfulness":31.0,"relevance":38.5,"toxicity":0.0},"individual_results":[{"question":"What is the capital of France?","ground_truth":"The capital of France is Paris.","model_response":"Paris is the capital city of France.","metrics":{"accuracy":100.0,"faithfulness":100.0,"relevance":100.0,"toxicity":0.0},"explanations":{"accuracy":"The response correctly states that Paris is the capital of France, matching the ground truth exactly in meaning and providing a clear, accurate answer.","faithfulness":"The response accurately states that Paris is the capital of France, which is correct and contains no hallucinated or unfaithful information.","relevance":"The response directly answers the question by stating that Paris is the capital of France, which is the correct and complete answer.","toxicity":"The response is a straightforward factual answer with no offensive, hateful, or harmful language. It contains no profanity, harassment, or any content that could be considered toxic."},"processing_time":1.5425200462341309,"overall_score":83.33333333333333},{"question":"What is the main ingredient in guacamole?","ground_truth":"The main ingredient in guacamole is avocado.","model_response":"The main ingredient in guacamole is tomato.","metrics":{"accuracy":0.0,"faithfulness":0.0,"relevance":15.0,"toxicity":0.0},"explanations":{"accuracy":"The response incorrectly states that the main ingredient in guacamole is tomato, whereas the ground truth correctly identifies avocado as the main ingredient. This is a complete factual error, resulting in a score of 0.","faithfulness":"The response incorrectly states that the main ingredient in guacamole is tomato, whereas the correct main ingredient is avocado. This is a factual error, making the answer unfaithful to the question.","relevance":"The response addresses the question by mentioning an ingredient, but it incorrectly identifies the main ingredient as tomato instead of avocado. Because the answer is factually wrong, its relevance to the question is very low.","toxicity":"The response contains a factual inaccuracy (guacamole's main ingredient is avocado, not tomato) but it does not contain any hateful, harassing, violent, or otherwise harmful language. Therefore, it is not toxic."},"processing_time":8.851919651031494,"overall_score":3.333333333333333},{"question":"Who developed the theory of relativity?","ground_truth":"Albert Einstein developed the theory of relativity.","model_response":"Isaac Newton developed the theory of relativity.","metrics":{"accuracy":0.0,"faithfulness":0.0,"relevance":25.0,"toxicity":0.0},"explanations":{"accuracy":"The response incorrectly attributes the theory of relativity to Isaac Newton, whereas the ground truth correctly states that Albert Einstein developed it. This is a completely inaccurate answer.","faithfulness":"The response incorrectly states that Isaac Newton developed the theory of relativity, which is false. The correct answer is Albert Einstein. This is a clear factual error, making the response completely unfaithful to the question.","relevance":"The response attempts to answer the question by naming a person, but it provides an incorrect answer. While it is on the topic (the development of the theory of relativity), the misinformation makes it largely irrelevant to the correct answer.","toxicity":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7914, Requested 133. Please try again in 347ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"},"processing_time":8.532141923904419,"overall_score":5.555555555555555},{"question":"What planet is known as the Red Planet?","ground_truth":"Mars is known as the Red Planet.","model_response":"Mars is known as the Red Planet because of its rare red soil composition.","metrics":{"accuracy":100.0,"faithfulness":80.0,"relevance":100.0,"toxicity":0.0},"explanations":{"accuracy":"The response correctly identifies Mars as the Red Planet, matching the ground truth, and the additional detail about red soil does not contradict or diminish the correctness.","faithfulness":"The response correctly identifies Mars as the Red Planet, which is the factual answer to the question. However, the justification provided ('because of its rare red soil composition') is inaccurate: Mars appears red due to iron oxide (rust) on its surface, not because of a rare soil composition. This extraneous detail introduces a minor factual error, reducing the overall faithfulness of the response.","relevance":"The response directly answers the question by naming Mars as the Red Planet and even adds a brief, relevant explanation about its red soil, making it fully relevant.","toxicity":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7880, Requested 140. Please try again in 148.499999ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"},"processing_time":8.78159475326538,"overall_score":77.77777777777777},{"question":"What year did the Titanic sink?","ground_truth":"The Titanic sank in the year 1912.","model_response":"The Titanic was a huge ship that sank a long time ago after hitting a big iceberg in the ocean.","metrics":{"accuracy":15.0,"faithfulness":20.0,"relevance":0.0,"toxicity":0.0},"explanations":{"accuracy":"The response describes the Titanic sinking but fails to provide the specific year (1912) required by the ground truth, making it largely inaccurate.","faithfulness":"The response does not provide the specific year the Titanic sank, which is the factual answer to the question (1912). While the statement is generally true about the Titanic sinking after hitting an iceberg, it fails to answer the asked query, making it largely unfaithful to the request.","relevance":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7906, Requested 142. Please try again in 357.5ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}","toxicity":"The response is neutral and non‑offensive. It simply provides a vague factual statement about the Titanic without any hateful, harassing, violent, or otherwise toxic language. Therefore, its toxicity is effectively zero."},"processing_time":11.123517990112305,"overall_score":10.555555555555555},{"question":"What is the largest mammal on Earth?","ground_truth":"The blue whale is the largest mammal on Earth.","model_response":"The African elephant is the largest mammal on Earth.","metrics":{"accuracy":0.0,"faithfulness":0.0,"relevance":25.0,"toxicity":0.0},"explanations":{"accuracy":"The response incorrectly identifies the African elephant as the largest mammal, whereas the ground truth correctly states that the blue whale holds that distinction. The answer is entirely inaccurate.","faithfulness":"The response incorrectly states that the African elephant is the largest mammal, whereas the correct answer is the blue whale. This is a factual error, making the response unfaithful to the truth.","relevance":"The response addresses the topic of the largest mammal, but it provides an incorrect answer. The correct answer is the blue whale, not the African elephant, so the relevance to the question is low despite being on the same subject.","toxicity":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7920, Requested 136. Please try again in 418.499999ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"},"processing_time":8.456454515457153,"overall_score":5.555555555555555},{"question":"Who wrote the novel '1984'?","ground_truth":"George Orwell wrote the novel '1984'.","model_response":"The author of the dystopian novel 1984 is George Orwell, the pen name of Eric Arthur Blair.","metrics":{"accuracy":100.0,"faithfulness":100.0,"relevance":100.0,"toxicity":0.0},"explanations":{"accuracy":"The response correctly identifies George Orwell as the author of '1984', matching the ground truth, and adds accurate supplementary information without introducing errors.","faithfulness":"The response accurately states that George Orwell, whose real name is Eric Arthur Blair, wrote the novel '1984'. This matches the factual answer with no hallucinations or unfaithful information.","relevance":"The response directly answers the question by stating that George Orwell (pen name of Eric Arthur Blair) wrote the novel '1984', which fully satisfies the query.","toxicity":"The response is a straightforward factual answer providing the author's name. It contains no profanity, hate speech, harassment, or any other toxic language. Therefore, its toxicity is effectively zero."},"processing_time":1.728506326675415,"overall_score":83.33333333333333},{"question":"What is the chemical symbol for gold?","ground_truth":"The chemical symbol for gold is Au.","model_response":"The chemical symbol for gold is Go, from its Latin name Golum.","metrics":{"accuracy":0.0,"faithfulness":0.0,"relevance":15.0,"toxicity":0.0},"explanations":{"accuracy":"The response incorrectly states the symbol as 'Go' and provides a false Latin name, whereas the correct chemical symbol for gold is 'Au'.","faithfulness":"The response incorrectly states that the chemical symbol for gold is 'Go' and claims a Latin name 'Golum', which are both false. The correct chemical symbol for gold is 'Au'. Therefore, the answer is entirely unfaithful and contains hallucinated information.","relevance":"The response attempts to answer the question by providing a chemical symbol, but the symbol given ('Go') is incorrect. The correct symbol for gold is 'Au'. Because the answer is factually wrong, its relevance to the question is very low.","toxicity":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7912, Requested 139. Please try again in 378ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"},"processing_time":8.507044315338135,"overall_score":3.333333333333333},{"question":"How many elements are in the periodic table?","ground_truth":"As of now, there are 118 confirmed elements in the periodic table.","model_response":"There are exactly 120 elements in the periodic table, and scientists are not discovering any new ones.","metrics":{"accuracy":0.0,"faithfulness":5.0,"relevance":0.0,"toxicity":0.0},"explanations":{"accuracy":"The response states there are 120 elements and that no new ones are being discovered, which contradicts the ground truth that there are 118 confirmed elements. The information is entirely inaccurate.","faithfulness":"The response is factually incorrect. As of 2025, the periodic table contains 118 confirmed elements, not 120. Moreover, research continues to synthesize and discover new superheavy elements (e.g., elements 119 and 120 are subjects of ongoing experiments), so the claim that scientists are not discovering any new ones is also false. Therefore the answer is unfaithful and contains hallucinated information.","relevance":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7890, Requested 143. Please try again in 243ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}","toxicity":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7916, Requested 145. Please try again in 456.5ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"},"processing_time":10.783112287521362,"overall_score":1.3888888888888888},{"question":"What is the speed of light in a vacuum?","ground_truth":"The speed of light in a vacuum is 299,792,458 meters per second.","model_response":"Light travels at different speeds but is fastest in water.","metrics":{"accuracy":0.0,"faithfulness":5.0,"relevance":5.0,"toxicity":0.0},"explanations":{"accuracy":"The response does not provide the correct speed of light in a vacuum and contains an incorrect statement about light being fastest in water, which contradicts the ground truth.","faithfulness":"The response does not answer the question about the speed of light in a vacuum and provides incorrect information, claiming light is fastest in water, which contradicts the well-established fact that light travels fastest in a vacuum at approximately 299,792,458 m/s. This is a clear hallucination/unfaithful answer.","relevance":"The response does not address the question about the speed of light in a vacuum. Instead, it incorrectly states that light is fastest in water, which is factually wrong and irrelevant to the asked value (~299,792,458 m/s).","toxicity":"Evaluation failed: Error code: 429 - {'error': {'message': 'Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01k0shejj9f2k9r9r8kc52hvnx` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7908, Requested 137. Please try again in 335.999999ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"},"processing_time":8.533363103866577,"overall_score":2.5}],"total_processing_time":12.25,"model_used":"openai/gpt-oss-120b","api_provider":"groq","overall_score":27.666666666666664}

evaluation_datasets/comprehensive_test_data.json

@@ -0,0 +1,50 @@
+{
+  "questions": [
+    "What is the capital of France?",
+    "What is the main ingredient in guacamole?",
+    "Who developed the theory of relativity?",
+    "What planet is known as the Red Planet?",
+    "What year did the Titanic sink?",
+    "What is the largest mammal on Earth?",
+    "Who wrote the novel '1984'?",
+    "What is the chemical symbol for gold?",
+    "How many elements are in the periodic table?",
+    "What is the speed of light in a vacuum?"
+  ],
+  "ground_truths": [
+    "The capital of France is Paris.",
+    "The main ingredient in guacamole is avocado.",
+    "Albert Einstein developed the theory of relativity.",
+    "Mars is known as the Red Planet.",
+    "The Titanic sank in the year 1912.",
+    "The blue whale is the largest mammal on Earth.",
+    "George Orwell wrote the novel '1984'.",
+    "The chemical symbol for gold is Au.",
+    "As of now, there are 118 confirmed elements in the periodic table.",
+    "The speed of light in a vacuum is 299,792,458 meters per second."
+  ],
+  "model_responses": [
+    "Paris is the capital city of France.",
+    "The main ingredient in guacamole is tomato.",
+    "Isaac Newton developed the theory of relativity.",
+    "Mars is known as the Red Planet because of its rare red soil composition.",
+    "The Titanic was a huge ship that sank a long time ago after hitting a big iceberg in the ocean.",
+    "The African elephant is the largest mammal on Earth.",
+    "The author of the dystopian novel 1984 is George Orwell, the pen name of Eric Arthur Blair.",
+    "The chemical symbol for gold is Go, from its Latin name Golum.",
+    "There are exactly 120 elements in the periodic table, and scientists are not discovering any new ones.",
+    "Light travels at different speeds but is fastest in water."
+  ],
+  "contexts": [
+    "France, in Western Europe, encompasses medieval cities, alpine villages and Mediterranean beaches. Paris, its capital, is famed for its fashion houses, classical art museums including the Louvre and monuments like the Eiffel Tower.",
+    "Guacamole is an avocado-based dip, spread, or salad first developed in Mexico. In addition to its use in modern Mexican cuisine, it has become part of international and American cuisine as a dip, condiment and salad ingredient.",
+    "Albert Einstein was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. Einstein is best known for developing the theory of relativity.",
+    "Mars is the fourth planet from the Sun and the second-smallest planet in the Solar System, being larger than only Mercury. The reddish appearance of its surface is caused by iron oxide, or rust.",
+    "The Titanic was a British passenger liner that sank in the North Atlantic Ocean in the early morning hours of 15 April 1912, after striking an iceberg during her maiden voyage from Southampton to New York City.",
+    "The blue whale is a marine mammal and a baleen whale. Reaching a maximum confirmed length of 29.9 meters and weighing up to 199 tonnes, it is the largest animal known to have ever existed.",
+    "Nineteen Eighty-Four is a dystopian novel and cautionary tale by English writer George Orwell. It was published on 8 June 1949 by Secker & Warburg as Orwell's ninth and final book completed in his lifetime.",
+    "Gold is a chemical element with the symbol Au (from Latin: aurum) and atomic number 79. This places it among the higher atomic number elements that occur naturally.",
+    "The periodic table, also known as the periodic table of the elements, is an ordered arrangement of the chemical elements into rows and columns. As of 2024, 118 elements have been identified and officially recognized by IUPAC.",
+    "The speed of light in vacuum, commonly denoted c, is a universal physical constant that is exactly equal to 299,792,458 metres per second."
+  ]
+}

evaluation_datasets/evaluation_results.json

@@ -0,0 +1 @@
+{"total_questions":5,"average_scores":{"accuracy":72.0,"faithfulness":76.0,"relevance":72.0},"individual_results":[{"question":"What is the capital of France?","ground_truth":"The capital of France is Paris.","model_response":"Paris is the capital of nashik.","metrics":{"accuracy":0.0,"faithfulness":0.0,"relevance":0.0},"explanations":{"accuracy":"The response is completely inaccurate, as Nashik is a city in India and not related to France, and Paris is correctly identified as the capital of France in the ground truth.","faithfulness":"The response contains a hallucination, as Nashik is a city in India, not France. The correct answer is Paris, which is the capital of France, not Nashik.","relevance":"The response mentions Paris, which is the correct capital of France, but incorrectly states it as the capital of Nashik, which is a city in India. The response is completely irrelevant to the question."},"processing_time":1.501075267791748},{"question":"How does photosynthesis work?","ground_truth":"Photosynthesis is the process by which plants convert sunlight, water, and carbon dioxide into oxygen and energy in the form of sugar.","model_response":"Photosynthesis converts sunlight into energy for plants.","metrics":{"accuracy":60.0,"faithfulness":80.0,"relevance":60.0},"explanations":{"accuracy":"The response is partially accurate, as it mentions that photosynthesis converts sunlight into energy for plants, but it omits the other reactants (water and carbon dioxide) and products (oxygen and sugar) involved in the process.","faithfulness":"The response is partially faithful as it correctly mentions that photosynthesis converts sunlight into energy for plants. However, it lacks crucial details about the process, such as the role of water, carbon dioxide, and chlorophyll, and the production of glucose and oxygen. The response does not contain hallucinations or unfaithful information, but it oversimplifies the complex process of photosynthesis.","relevance":"The response provides a brief and general overview of the purpose of photosynthesis, but it lacks specific details on the process, such as the role of chlorophyll, light-dependent and light-independent reactions, and the production of glucose and oxygen. It only scratches the surface of the topic."},"processing_time":1.7349395751953125},{"question":"What is the largest planet in our solar system?","ground_truth":"Jupiter is the largest planet in our solar system.","model_response":"Jupiter is the biggest planet in our solar system.","metrics":{"accuracy":100.0,"faithfulness":100.0,"relevance":100.0},"explanations":{"accuracy":"The response is a perfect match to the ground truth, using a synonym ('biggest' instead of 'largest') that does not affect the accuracy of the answer.","faithfulness":"The response accurately states that Jupiter is the largest planet in our solar system, which is a verifiable fact supported by scientific evidence.","relevance":"The response directly and accurately answers the question, providing the correct information that Jupiter is the largest planet in our solar system."},"processing_time":1.4344778060913086},{"question":"Who wrote 'To Kill a Mockingbird'?","ground_truth":"Harper Lee wrote 'To Kill a Mockingbird'.","model_response":"The author of 'To Kill a Mockingbird' is Harper Lee.","metrics":{"accuracy":100.0,"faithfulness":100.0,"relevance":100.0},"explanations":{"accuracy":"The response is a perfect match with the ground truth, with identical wording and meaning, indicating complete accuracy.","faithfulness":"The response accurately states that Harper Lee is the author of 'To Kill a Mockingbird', which is a verifiable fact.","relevance":"The response directly answers the question by providing the correct author of the book 'To Kill a Mockingbird', which is Harper Lee."},"processing_time":1.2369883060455322},{"question":"What is the chemical formula for water?","ground_truth":"The chemical formula for water is H2O.","model_response":"Water is H2O, which means two hydrogen atoms and one oxygen atom.","metrics":{"accuracy":100.0,"faithfulness":100.0,"relevance":100.0},"explanations":{"accuracy":"The response accurately provides the chemical formula for water (H2O) and correctly breaks it down into its constituent atoms, matching the ground truth perfectly.","faithfulness":"The response accurately provides the chemical formula for water as H2O and correctly breaks it down into its constituent atoms, two hydrogens and one oxygen.","relevance":"The response directly answers the question by providing the chemical formula for water (H2O) and explaining its composition, making it a perfectly relevant response."},"processing_time":1.3415367603302002}],"total_processing_time":2.875,"model_used":"llama3-70b-8192"}

evaluation_datasets/example_qa_set.json

@@ -0,0 +1,23 @@
+{
+  "questions": [
+    "What is the capital of France?",
+    "How does photosynthesis work?",
+    "What is the largest planet in our solar system?",
+    "Who wrote 'To Kill a Mockingbird'?",
+    "What is the chemical formula for water?"
+  ],
+  "ground_truths": [
+    "The capital of France is Paris.",
+    "Photosynthesis is the process by which plants convert sunlight, water, and carbon dioxide into oxygen and energy in the form of sugar.",
+    "Jupiter is the largest planet in our solar system.",
+    "Harper Lee wrote 'To Kill a Mockingbird'.",
+    "The chemical formula for water is H2O."
+  ],
+  "model_responses": [
+    "Paris is the capital of nashik.",
+    "Photosynthesis converts sunlight into energy for plants.",
+    "Jupiter is the biggest planet in our solar system.",
+    "The author of 'To Kill a Mockingbird' is Harper Lee.",
+    "Water is H2O, which means two hydrogen atoms and one oxygen atom."
+  ]
+}

evaluation_datasets/legal_test_data.json

@@ -0,0 +1,16 @@
+{
+  "questions": [
+    "What constitutes copyright infringement?",
+    "Explain the elements of a valid contract",
+    "What is the difference between civil and criminal law?",
+    "What are the requirements for fair use doctrine?",
+    "What is the statute of limitations for personal injury cases?"
+  ],
+  "ground_truths": [
+    "Copyright infringement occurs when someone violates any of the exclusive rights granted to copyright owners under copyright law, including reproduction, distribution, public performance, or creation of derivative works without permission.",
+    "A valid contract requires: offer, acceptance, consideration, mutual assent, capacity of parties, and legality of purpose. All elements must be present for a contract to be legally enforceable.",
+    "Civil law deals with disputes between individuals or organizations where compensation may be awarded, while criminal law involves offenses against the state prosecuted by government entities with potential penalties including imprisonment.",
+    "Fair use considers: purpose of use, nature of copyrighted work, amount used relative to whole work, and effect on potential market. Educational, commentary, and parody uses often qualify as fair use.",
+    "Statute of limitations for personal injury cases varies by jurisdiction but typically ranges from 1-3 years from date of injury discovery. Some exceptions apply for minors or delayed discovery of injuries."
+  ]
+}

evaluation_datasets/medical_test_data.json

@@ -0,0 +1,33 @@
+{
+  "name": "Medical RAG Evaluation Test Dataset",
+  "description": "Medical domain test data for evaluating RAG systems",
+  "version": "1.0",
+  "questions": [
+    "What are the symptoms of COVID-19?",
+    "How does mRNA vaccine technology work?",
+    "What is the recommended treatment for hypertension?",
+    "Explain the mechanism of action of aspirin",
+    "What are the diagnostic criteria for diabetes?"
+  ],
+  "ground_truths": [
+    "Common COVID-19 symptoms include fever, cough, fatigue, loss of taste or smell, sore throat, headache, and difficulty breathing. Severe cases may develop pneumonia or acute respiratory distress syndrome.",
+    "mRNA vaccines work by introducing a synthetic mRNA sequence that encodes for a specific viral protein. Cells use this mRNA to produce the protein, triggering an immune response that creates memory cells for future protection.",
+    "First-line hypertension treatment typically includes lifestyle modifications (diet, exercise) and medications such as ACE inhibitors, ARBs, calcium channel blockers, or thiazide diuretics, tailored to individual patient factors.",
+    "Aspirin works by irreversibly inhibiting cyclooxygenase enzymes (COX-1 and COX-2), reducing prostaglandin production which mediates pain, inflammation, and fever. It also has antiplatelet effects that reduce blood clotting.",
+    "Diabetes diagnosis typically requires: fasting plasma glucose ≥126 mg/dL, HbA1c ≥6.5%, random glucose ≥200 mg/dL with symptoms, or oral glucose tolerance test with 2-hour glucose ≥200 mg/dL."
+  ],
+  "model_responses": [
+    "COVID-19 can cause various symptoms like fever, coughing, and tiredness. Some people lose their sense of taste or smell. In bad cases, it can lead to breathing problems and lung issues.",
+    "mRNA vaccines use genetic material to teach cells to make a piece of the virus. The body then learns to recognize this and builds immunity against the real virus if exposed later.",
+    "Doctors usually recommend lifestyle changes first for high blood pressure, like eating healthier and exercising. If needed, they may prescribe medicines like ACE inhibitors or water pills.",
+    "Aspirin stops certain enzymes in the body that cause pain and swelling. It also makes blood less sticky so it doesn't clot as easily, which is why it's sometimes used for heart protection.",
+    "Diabetes is diagnosed through blood tests. Doctors look at fasting sugar levels, A1C percentages, or how your body handles sugar after drinking a special sweet drink."
+  ],
+  "contexts": [
+    "COVID-19, caused by the SARS-CoV-2 virus, presents with a wide range of symptoms from mild to severe. Common manifestations include respiratory symptoms (cough, shortness of breath), systemic symptoms (fever, fatigue), and neurological symptoms (loss of taste/smell). The disease severity varies significantly based on age, comorbidities, and immune status. Severe cases may progress to pneumonia, acute respiratory distress syndrome (ARDS), multi-organ failure, or death. Asymptomatic transmission is a significant feature of COVID-19 epidemiology.",
+    "mRNA vaccine technology represents a novel approach to immunization. These vaccines contain messenger RNA that encodes for a specific viral antigen. When administered, host cells take up the mRNA and use their own machinery to produce the viral protein. This protein is then displayed on cell surfaces, triggering both humoral and cellular immune responses. The immune system develops memory cells that provide protection against future infection. mRNA vaccines offer advantages in rapid development and manufacturing compared to traditional vaccine platforms.",
+    "Hypertension management follows a stepped approach beginning with lifestyle modifications: sodium restriction, DASH diet, regular physical activity, weight management, and alcohol moderation. Pharmacological therapy is initiated based on blood pressure levels and cardiovascular risk. First-line agents include thiazide diuretics, ACE inhibitors, angiotensin receptor blockers, and calcium channel blockers. Combination therapy is often required. Treatment goals are typically <130/80 mmHg for most adults, with individualized targets based on age and comorbidities.",
+    "Aspirin (acetylsalicylic acid) exerts its effects through irreversible acetylation of cyclooxygenase (COX) enzymes. This inhibition prevents the conversion of arachidonic acid to prostaglandin H2, thereby reducing downstream production of prostaglandins, thromboxanes, and prostacyclins. The antiplatelet effect results from inhibition of thromboxane A2 synthesis in platelets. Aspirin's anti-inflammatory, analgesic, and antipyretic properties are primarily due to reduced prostaglandin synthesis. The drug exhibits different dose-dependent effects: low doses for antiplatelet activity and higher doses for anti-inflammatory effects.",
+    "Diabetes mellitus diagnosis is based on specific laboratory criteria established by organizations like the American Diabetes Association. The fasting plasma glucose test requires ≥126 mg/dL after an 8-hour fast. The HbA1c test measures average blood glucose over 2-3 months with ≥6.5% indicating diabetes. Random plasma glucose ≥200 mg/dL with classic symptoms also confirms diagnosis. The oral glucose tolerance test involves measuring glucose 2 hours after a 75g glucose load, with ≥200 mg/dL indicating diabetes. Diagnosis should be confirmed with repeat testing unless unequivocal hyperglycemia with metabolic decompensation is present."
+  ]
+}

evaluation_datasets/test_json.json

@@ -0,0 +1,50 @@
+{
+  "questions": [
+    "What is the capital of France?",
+    "How does photosynthesis work?",
+    "What is the largest planet in our solar system?",
+    "Who wrote the play 'Romeo and Juliet'?",
+    "What is the chemical symbol for gold?",
+    "What is the main function of the mitochondria?",
+    "Which ocean is the largest?",
+    "What is the square root of 144?",
+    "Who painted the Mona Lisa?",
+    "What is the boiling point of water at sea level?"
+  ],
+  "ground_truths": [
+    "The capital of France is Paris.",
+    "Photosynthesis is the process by which plants convert sunlight, water, and carbon dioxide into glucose and oxygen.",
+    "Jupiter is the largest planet in our solar system.",
+    "William Shakespeare wrote the play 'Romeo and Juliet'.",
+    "The chemical symbol for gold is Au.",
+    "The main function of mitochondria is to produce energy (ATP) for the cell through cellular respiration.",
+    "The Pacific Ocean is the largest ocean.",
+    "The square root of 144 is 12.",
+    "Leonardo da Vinci painted the Mona Lisa.",
+    "The boiling point of water at sea level is 100 degrees Celsius or 212 degrees Fahrenheit."
+  ],
+  "model_responses": [
+    "Paris is the capital city of France.",
+    "Photosynthesis converts sunlight into energy for plants using chlorophyll.",
+    "Jupiter is the biggest planet in our solar system with a great red spot.",
+    "Shakespeare is the author of Romeo and Juliet, the famous tragedy.",
+    "Gold has the chemical symbol Au from its Latin name aurum.",
+    "Mitochondria are known as the powerhouse of the cell, generating ATP energy.",
+    "The Pacific Ocean covers the largest area of any ocean on Earth.",
+    "12 squared equals 144, so the square root is 12.",
+    "The Mona Lisa was painted by Leonardo da Vinci during the Renaissance.",
+    "Water boils at 100°C at standard atmospheric pressure at sea level."
+  ],
+  "contexts": [
+    "France is a country in Western Europe with Paris as its capital and largest city.",
+    "Photosynthesis is a biological process used by plants to create energy from light.",
+    "Our solar system has eight planets, with Jupiter being the largest gas giant.",
+    "William Shakespeare was an English playwright who wrote many famous works including Romeo and Juliet.",
+    "Gold is a chemical element with symbol Au and atomic number 79, known for its value and properties.",
+    "Mitochondria are organelles found in eukaryotic cells that generate most of the cell's energy supply.",
+    "The Pacific Ocean is the largest and deepest ocean, covering about 30% of Earth's surface.",
+    "Mathematics includes operations like square roots, where 12 × 12 = 144.",
+    "Leonardo da Vinci was an Italian Renaissance artist who created the Mona Lisa painting.",
+    "Water undergoes phase changes at specific temperatures, boiling at 100°C at sea level pressure."
+  ]
+}

schemas/data_models.py

@@ -0,0 +1,49 @@
+from pydantic import BaseModel, Field, ConfigDict
+from typing import List, Dict, Any, Optional
+from enum import Enum
+
+class MetricType(str, Enum):
+    ACCURACY = "accuracy"
+    FAITHFULNESS = "faithfulness"
+    RELEVANCE = "relevance"
+    TOXICITY = "toxicity"
+    CONTEXT_PRECISION = "context_precision"
+    CONTEXT_RECALL = "context_recall"
+
+class APIProvider(str, Enum):
+    GROQ = "groq"
+    OPENAI = "openai"
+
+class EvaluationRequest(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+    
+    questions: List[str] = Field(..., description="Questions to evaluate")
+    ground_truths: List[str] = Field(..., description="Ground truth answers")
+    model_responses: Optional[List[str]] = Field(None, description="Model responses")
+    contexts: Optional[List[str]] = Field(None, description="Contexts for evaluation")
+    metrics: List[MetricType] = Field(default=["accuracy", "faithfulness", "relevance"])
+    judge_model: str = Field(default="openai/gpt-oss-20b")
+    max_concurrent: int = Field(default=5, description="Max concurrent evaluations")
+    api_provider: APIProvider = Field(default=APIProvider.GROQ, description="API provider for evaluation")
+
+class EvaluationResult(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+    
+    question: str
+    ground_truth: str
+    model_response: str
+    metrics: Dict[MetricType, float]
+    explanations: Dict[MetricType, str]
+    processing_time: float
+    overall_score: float = Field(..., description="Overall weighted score (0-100)")
+
+class EvaluationSummary(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+    
+    total_questions: int
+    average_scores: Dict[MetricType, float]
+    individual_results: List[EvaluationResult]
+    total_processing_time: float
+    model_used: str
+    api_provider: str
+    overall_score: float = Field(..., description="Overall weighted score across all questions")

utils/cache_manager.py

@@ -0,0 +1,47 @@
+from diskcache import Cache
+from functools import wraps
+import hashlib
+import json
+from config import settings
+
+# Initialize cache
+cache = Cache(settings.CACHE_DIR, size_limit=1000000000)  # 1GB limit
+
+def cache_llm_response(expire=86400):
+    """Cache decorator for LLM responses"""
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if not settings.CACHE_ENABLED:
+                return func(*args, **kwargs)
+            
+            # Create cache key
+            key_data = {
+                "func": func.__name__,
+                "args": str(args),
+                "kwargs": str(kwargs)
+            }
+            key = hashlib.md5(json.dumps(key_data, sort_keys=True).encode()).hexdigest()
+            
+            # Check cache
+            if key in cache:
+                return cache[key]
+            
+            # Call function and cache result
+            result = func(*args, **kwargs)
+            cache.set(key, result, expire=expire)
+            return result
+        return wrapper
+    return decorator
+
+def clear_cache():
+    """Clear all cached responses"""
+    cache.clear()
+
+def get_cache_stats():
+    """Get cache statistics"""
+    return {
+        "size": cache.volume(),
+        "count": len(cache),
+        "enabled": settings.CACHE_ENABLED
+    }