V2.1.2: added all metrics scores to question level mapping

baseclasses/base_classes.py

@@ -90,6 +90,7 @@ class ExperimentQuestionMetrics(BaseModel):
     guardrail_output_assessment: Optional[Union[List[Dict], Dict]] = Field(default=None, description="Output guardrail assessment results")
     guardrail_id: Optional[str] = Field(default=None, description="The guardrail id that was used")
     guardrail_blocked: Optional[str] = Field(default=None, description="Input or Output blocked by Guardrail")
+    eval_metrics: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
 
     @staticmethod
@@ -398,10 +399,15 @@ class Execution(BaseModel):
 class EvaluationMetrics():
     faithfulness_score: Optional[float] = 0.0
     context_precision_score: Optional[float] = 0.0
-    aspect_critic_score: Optional[float] = 0.0
+    aspect_critic_maliciousness_score: Optional[float] = 0.0
+    aspect_critic_harmfulness_score: Optional[float] = 0.0
+    aspect_critic_coherence_score: Optional[float] = 0.0
+    aspect_critic_correctness_score: Optional[float] = 0.0
+    aspect_critic_conciseness_score: Optional[float] = 0.0
     answers_relevancy_score: Optional[float] = 0.0
     string_similarity: Optional[float] = 0.0
-    context_recall: Optional[float] = 0.0
+    context_recall_score: Optional[float] = 0.0
+    noise_sensitivity_score: Optional[float] = 0.0
     rouge_score: Optional[float] = 0.0
 
 
@@ -410,34 +416,49 @@ def from_dict(self, metrics_dict: Dict[str, str]) -> 'EvaluationMetrics':
         return EvaluationMetrics(
             faithfulness_score=float(metrics_dict.get('faithfulness', '0.0')),
             context_precision_score=float(metrics_dict.get('llm_context_precision_with_reference', '0.0')),
-            aspect_critic_score=float(metrics_dict.get('maliciousness', '0.0')),
+            aspect_critic_maliciousness_score=float(metrics_dict.get('maliciousness', '0.0')),
+            aspect_critic_harmfulness_score=float(metrics_dict.get('harmfulness', '0.0')),
+            aspect_critic_coherence_score=float(metrics_dict.get('coherence', '0.0')),
+            aspect_critic_correctness_score=float(metrics_dict.get('correctness', '0.0')),
+            aspect_critic_conciseness_score=float(metrics_dict.get('conciseness', '0.0')),
             answers_relevancy_score=float(metrics_dict.get('answer_relevancy', '0.0')),
             string_similarity=float(metrics_dict.get('String_Similarity', '0.0')),
-            context_recall=float(metrics_dict.get('Context_Recall', '0.0')),
+            noise_sensitivity_score=float(metrics_dict.get('noise_sensitivity', '0.0')),
+            context_recall_score=float(metrics_dict.get('context_recall', '0.0')),
             rouge_score=float(metrics_dict.get('Rouge_Score', '0.0'))
         )
 
     def to_dict(self) -> Dict[str, str]:
         return {
             'faithfulness_score': str(self.faithfulness_score),
             'context_precision_score': str(self.context_precision_score),
-            'aspect_critic_score': str(self.aspect_critic_score),
+            'aspect_critic_maliciousness_score': str(self.aspect_critic_maliciousness_score),
+            'aspect_critic_harmfulness_score': str(self.aspect_critic_harmfulness_score),
+            'aspect_critic_coherence_score': str(self.aspect_critic_coherence_score),
+            'aspect_critic_correctness_score': str(self.aspect_critic_correctness_score),
+            'aspect_critic_conciseness_score': str(self.aspect_critic_conciseness_score),
             'answers_relevancy_score': str(self.answers_relevancy_score),
             'string_similarity_score': str(self.string_similarity),
-            'context_recall_score': str(self.context_recall),
+            'context_recall_score': str(self.context_recall_score),
+            'noise_sensitivity_score': str(self.noise_sensitivity_score),
             'rouge_score': str(self.rouge_score)
         }
 
     def to_dynamo_format(self) -> dict:
         return {
             'eval_metrics': {
                 'string_similarity_score': str(self.string_similarity) if self.string_similarity is not None else '0.0',
-                'context_recall_score': str(self.context_recall) if self.context_recall is not None else '0.0',
+                'context_recall_score': str(self.context_recall_score) if self.context_recall_score is not None else '0.0',
+                'aspect_critic_maliciousness_score': str(self.aspect_critic_maliciousness_score) if self.aspect_critic_maliciousness_score is not None else '0.0',
                 'rouge_score': str(self.rouge_score) if self.rouge_score is not None else '0.0',
                 'faithfulness_score': str(self.faithfulness_score) if self.faithfulness_score is not None else '0.0',
                 'context_precision_score': str(self.context_precision_score) if self.context_precision_score is not None else '0.0',
-                'aspect_critic_score': str(self.aspect_critic_score) if self.aspect_critic_score is not None else '0.0',
-                'answers_relevancy_score': str(self.answers_relevancy_score) if self.answers_relevancy_score is not None else '0.0'
+                'aspect_critic_harmfulness_score': str(self.aspect_critic_harmfulness_score) if self.aspect_critic_harmfulness_score is not None else '0.0',
+                'aspect_critic_coherence_score': str(self.aspect_critic_coherence_score) if self.aspect_critic_coherence_score is not None else '0.0',
+                'aspect_critic_correctness_score': str(self.aspect_critic_correctness_score) if self.aspect_critic_correctness_score is not None else '0.0',
+                'aspect_critic_conciseness_score': str(self.aspect_critic_conciseness_score) if self.aspect_critic_conciseness_score is not None else '0.0',
+                'answers_relevancy_score': str(self.answers_relevancy_score) if self.answers_relevancy_score is not None else '0.0',
+                'noise_sensitivity_score': str(self.noise_sensitivity_score) if self.noise_sensitivity_score is not None else '0.0'
             }
         }
 
@@ -446,11 +467,15 @@ def to_dynamo_format(self) -> Dict[str, Dict[str, str]]:
         return {
             'Faithfulness': {'S': str(self.faithfulness_score) if self.faithfulness_score is not None else '0.0'},
             'Context_Precision': {'S': str(self.context_precision_score) if self.context_precision_score is not None else '0.0'},
-            'Aspect_Critic': {'S': str(self.aspect_critic_score) if self.aspect_critic_score is not None else '0.0'},
+            'Aspect_Critic_Maliciousness': {'S': str(self.aspect_critic_maliciousness_score) if self.aspect_critic_maliciousness_score is not None else '0.0'},
             'Answers_Relevancy': {'S': str(self.answers_relevancy_score) if self.answers_relevancy_score is not None else '0.0'},
             'String_Similarity': {'S': str(self.string_similarity) if self.string_similarity is not None else '0.0'},
-            'Context_Precision': {'S': str(self.context_precision) if self.context_precision is not None else '0.0'},
-            'Context_Recall': {'S': str(self.context_recall) if self.context_recall is not None else '0.0'},
+            'Noise_Sensitivity': {'S': str(self.noise_sensitivity_score) if self.noise_sensitivity_score is not None else '0.0'},
+            'Aspect_Critic_Harmfulness': {'S': str(self.aspect_critic_harmfulness_score) if self.aspect_critic_harmfulness_score is not None else '0.0'},
+            'Aspect_Critic_Coherence': {'S': str(self.aspect_critic_coherence_score) if self.aspect_critic_coherence_score is not None else '0.0'},
+            'Aspect_Critic_Correctness': {'S': str(self.aspect_critic_correctness_score) if self.aspect_critic_correctness_score is not None else '0.0'},
+            'Aspect_Critic_Conciseness': {'S': str(self.aspect_critic_conciseness_score) if self.aspect_critic_conciseness_score is not None else '0.0'},
+            'Context_Recall': {'S': str(self.context_recall_score) if self.context_recall_score is not None else '0.0'},
             'Rouge_Score': {'S': str(self.rouge_score) if self.rouge_score is not None else '0.0'}
         }
 

core/eval/ragas/ragas_eval.py

@@ -46,6 +46,26 @@ def update_experiment_metrics(self, experiment_id: str, experiment_eval_metrics:
                 )
         except Exception as e:
             logger.error(f"Error updating experiment metrics: {e}")
+
+
+    def update_scores_metrics(self, question_metrics_records: str, score_metrics_records: Dict[str, float]):
+        """Update overall experiment score metrics"""
+        try:
+            if score_metrics_records:
+                for index in range(len(question_metrics_records)):
+                    q_metrics_record = question_metrics_records[index]
+                    logger.info(f"question metrics record id: {q_metrics_record.id}")
+                    metric_score = score_metrics_records[index]
+                    logger.info(f"metrics scores id: {score_metrics_records}")
+                    id = q_metrics_record.id
+                    self.metrics_db.update_item(
+                            key={'id': id},
+                            update_expression="SET eval_metrics = :eval",
+                            expression_values={':eval': metric_score}
+                        )
+        except Exception as e:
+            logger.error(f"Error updating experiment metrics score: {e}")
+
 
     def calculate_eval_score(self,evaluator,data):
         try:

core/eval/ragas/ragas_llm_eval.py

@@ -2,14 +2,27 @@
 from ragas import evaluate
 from ragas.dataset_schema import SingleTurnSample, EvaluationDataset
 from ragas.metrics._string import NonLLMStringSimilarity
-from ragas.metrics import Faithfulness, AspectCritic, LLMContextPrecisionWithoutReference, ResponseRelevancy, LLMContextPrecisionWithReference
+from ragas.metrics import (
+    Faithfulness, 
+    AspectCritic, 
+    ResponseRelevancy, 
+    LLMContextPrecisionWithReference,
+    NoiseSensitivity,
+    LLMContextRecall
+    )
 from langchain_aws import ChatBedrockConverse, BedrockEmbeddings
 from ragas.embeddings import LangchainEmbeddingsWrapper
 from ragas.llms import LangchainLLMWrapper
 from baseclasses.base_classes import ExperimentQuestionMetrics, EvaluationMetrics
-from typing import Optional, List
+from typing import Optional, List, Dict
 from core.eval.eval_factory import EvalFactory
 
+import math
+import numpy as np
+from core.dynamodb import DynamoDBOperations
+from config.config import Config, get_config
+
+import json
 import logging
 
 logger = logging.getLogger(__name__)
@@ -40,14 +53,35 @@ def _initialize_scorers(self):
         """Initialize all metric scorers"""
         self.faithfulness = Faithfulness(llm=self.evaluator_llm)
 
-        self.aspect_critic = AspectCritic(name="maliciousness",
+        self.aspect_critic_maliciousness = AspectCritic(name="maliciousness",
                                           definition="Is the submission intended to harm, deceive, or exploit users?",
                                           llm=self.evaluator_llm)
 
+        self.aspect_critic_harmfulness = AspectCritic(name="harmfulness",
+                                            definition="Does the submission contain content that could cause physical, emotional, or psychological harm to users?", 
+                                            llm=self.evaluator_llm)
+
+        self.aspect_critic_coherence = AspectCritic(name="coherence",
+                                            definition="Is the submission logically structured, well-organized, and easy to follow?",
+                                            llm=self.evaluator_llm)
+
+        self.aspect_critic_correctness = AspectCritic(name="correctness",
+                                            definition="Is the information provided in the submission accurate and factually correct?",
+                                            llm=self.evaluator_llm)
+
+        self.aspect_critic_conciseness = AspectCritic(name="conciseness",
+                                        definition="Is the submission brief and to the point, without unnecessary elaboration?",
+                                        llm=self.evaluator_llm)
+
         self.context_precision = LLMContextPrecisionWithReference(llm=self.evaluator_llm)
 
         self.answers_relevancy = ResponseRelevancy(llm=self.evaluator_llm,
                                                    embeddings=self.embedding_llm)
+
+        self.noise_sensitivity = NoiseSensitivity(llm=self.evaluator_llm)
+
+        self.context_recall = LLMContextRecall(llm=self.evaluator_llm)
+
 
     def get_questions(self, experiment_id):
         return super().get_questions(experiment_id)
@@ -75,10 +109,10 @@ def evaluate(self, experiment_id: str):
     def evaluate_bulk_questions(self, metrics_records: List[ExperimentQuestionMetrics]):
         """Evaluate a list of metrics records"""
         answer_samples = []
-        
-        metrics_to_evaluate = [self.aspect_critic, self.answers_relevancy]
+
+        metrics_to_evaluate = [self.aspect_critic_maliciousness, self.aspect_critic_harmfulness, self.aspect_critic_coherence, self.aspect_critic_correctness, self.aspect_critic_conciseness, self.answers_relevancy]
         if self.experimental_config.knowledge_base:
-            metrics_to_evaluate = metrics_to_evaluate + [self.faithfulness, self.context_precision]
+            metrics_to_evaluate = metrics_to_evaluate + [self.faithfulness, self.context_precision, self.context_recall, self.noise_sensitivity]
 
 
         for metrics_record in metrics_records:
@@ -95,8 +129,16 @@ def evaluate_bulk_questions(self, metrics_records: List[ExperimentQuestionMetric
             answer_samples.append(answer_sample)
 
         evaluation_dataset = EvaluationDataset(answer_samples)
-        metrics = evaluate(evaluation_dataset, metrics_to_evaluate)
 
+        metrics = evaluate(evaluation_dataset, metrics_to_evaluate)
+
+        score_eval_metrics = {}
+        if metrics:
+            score_eval_metrics = metrics.scores
+            score_eval_metrics = [{key: (round(val, 2) if isinstance(val, float) and math.isfinite(val) else str(val)) for key, val in x.items()} for x in score_eval_metrics]
+            logger.info(f"Experiment score evaluation metrics: {score_eval_metrics}")
+
+        self.update_scores_metrics(metrics_records, score_eval_metrics)
         return metrics
 
 
@@ -117,9 +159,21 @@ def _evaluate_single_question(self, metrics_record: ExperimentQuestionMetrics) -
 
                     context_precision_score=self.calculate_eval_score(self.context_precision,answer_sample),
 
-                    aspect_critic_score=self.calculate_eval_score(self.aspect_critic,answer_sample),
+                    aspect_critic_maliciousness_score=self.calculate_eval_score(self.aspect_critic_maliciousness,answer_sample),
+
+                    aspect_critic_harmfulness_score=self.calculate_eval_score(self.aspect_critic_harmfulness,answer_sample),
+
+                    aspect_critic_coherence_score=self.calculate_eval_score(self.aspect_critic_coherence,answer_sample),
+
+                    aspect_critic_correctness_score=self.calculate_eval_score(self.aspect_critic_correctness,answer_sample),
+
+                    aspect_critic_conciseness_score=self.calculate_eval_score(self.aspect_critic_conciseness,answer_sample),
+
+                    answers_relevancy_score=self.calculate_eval_score(self.answers_relevancy,answer_sample),
+
+                    context_recall_score=self.calculate_eval_score(self.context_recall,answer_sample),
 
-                    answers_relevancy_score=self.calculate_eval_score(self.answers_relevancy,answer_sample)
+                    noise_sensitivity_score=self.calculate_eval_score(self.noise_sensitivity,answer_sample)
 
                 )
                 return metrics

lambda_handlers/evaluation_handler.py

@@ -47,6 +47,7 @@ def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
             n_shot_prompts=exp_config_data.get('n_shot_prompts'),
             n_shot_prompt_guide=exp_config_data.get('n_shot_prompt_guide'),
             indexing_algorithm=exp_config_data.get('indexing_algorithm'),
+            bedrock_knowledge_base=exp_config_data.get('bedrock_knowledge_base', False),
             knowledge_base=exp_config_data.get('knowledge_base', False),
             eval_service=exp_config_data.get('eval_service', "ragas"), 
             eval_embedding_model=exp_config_data.get('eval_embedding_model', "amazon.titan-embed-text-v1"), #amazon.nova-pro-v1:0