diff --git a/.gitignore b/.gitignore deleted file mode 100644 index fd5106ff..00000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.DS_STORE diff --git a/baseclasses/base_classes.py b/baseclasses/base_classes.py index 11337d2c..94d78145 100644 --- a/baseclasses/base_classes.py +++ b/baseclasses/base_classes.py @@ -90,6 +90,7 @@ class ExperimentQuestionMetrics(BaseModel): guardrail_output_assessment: Optional[Union[List[Dict], Dict]] = Field(default=None, description="Output guardrail assessment results") guardrail_id: Optional[str] = Field(default=None, description="The guardrail id that was used") guardrail_blocked: Optional[str] = Field(default=None, description="Input or Output blocked by Guardrail") + eval_metrics: Optional[Dict[str, Any]] = Field(default_factory=dict) @staticmethod @@ -398,10 +399,15 @@ class Execution(BaseModel): class EvaluationMetrics(): faithfulness_score: Optional[float] = 0.0 context_precision_score: Optional[float] = 0.0 - aspect_critic_score: Optional[float] = 0.0 + aspect_critic_maliciousness_score: Optional[float] = 0.0 + aspect_critic_harmfulness_score: Optional[float] = 0.0 + aspect_critic_coherence_score: Optional[float] = 0.0 + aspect_critic_correctness_score: Optional[float] = 0.0 + aspect_critic_conciseness_score: Optional[float] = 0.0 answers_relevancy_score: Optional[float] = 0.0 string_similarity: Optional[float] = 0.0 - context_recall: Optional[float] = 0.0 + context_recall_score: Optional[float] = 0.0 + noise_sensitivity_score: Optional[float] = 0.0 rouge_score: Optional[float] = 0.0 @@ -410,10 +416,15 @@ def from_dict(self, metrics_dict: Dict[str, str]) -> 'EvaluationMetrics': return EvaluationMetrics( faithfulness_score=float(metrics_dict.get('faithfulness', '0.0')), context_precision_score=float(metrics_dict.get('llm_context_precision_with_reference', '0.0')), - aspect_critic_score=float(metrics_dict.get('maliciousness', '0.0')), + aspect_critic_maliciousness_score=float(metrics_dict.get('maliciousness', '0.0')), + aspect_critic_harmfulness_score=float(metrics_dict.get('harmfulness', '0.0')), + aspect_critic_coherence_score=float(metrics_dict.get('coherence', '0.0')), + aspect_critic_correctness_score=float(metrics_dict.get('correctness', '0.0')), + aspect_critic_conciseness_score=float(metrics_dict.get('conciseness', '0.0')), answers_relevancy_score=float(metrics_dict.get('answer_relevancy', '0.0')), string_similarity=float(metrics_dict.get('String_Similarity', '0.0')), - context_recall=float(metrics_dict.get('Context_Recall', '0.0')), + noise_sensitivity_score=float(metrics_dict.get('noise_sensitivity', '0.0')), + context_recall_score=float(metrics_dict.get('context_recall', '0.0')), rouge_score=float(metrics_dict.get('Rouge_Score', '0.0')) ) @@ -421,10 +432,15 @@ def to_dict(self) -> Dict[str, str]: return { 'faithfulness_score': str(self.faithfulness_score), 'context_precision_score': str(self.context_precision_score), - 'aspect_critic_score': str(self.aspect_critic_score), + 'aspect_critic_maliciousness_score': str(self.aspect_critic_maliciousness_score), + 'aspect_critic_harmfulness_score': str(self.aspect_critic_harmfulness_score), + 'aspect_critic_coherence_score': str(self.aspect_critic_coherence_score), + 'aspect_critic_correctness_score': str(self.aspect_critic_correctness_score), + 'aspect_critic_conciseness_score': str(self.aspect_critic_conciseness_score), 'answers_relevancy_score': str(self.answers_relevancy_score), 'string_similarity_score': str(self.string_similarity), - 'context_recall_score': str(self.context_recall), + 'context_recall_score': str(self.context_recall_score), + 'noise_sensitivity_score': str(self.noise_sensitivity_score), 'rouge_score': str(self.rouge_score) } @@ -432,12 +448,17 @@ def to_dynamo_format(self) -> dict: return { 'eval_metrics': { 'string_similarity_score': str(self.string_similarity) if self.string_similarity is not None else '0.0', - 'context_recall_score': str(self.context_recall) if self.context_recall is not None else '0.0', + 'context_recall_score': str(self.context_recall_score) if self.context_recall_score is not None else '0.0', + 'aspect_critic_maliciousness_score': str(self.aspect_critic_maliciousness_score) if self.aspect_critic_maliciousness_score is not None else '0.0', 'rouge_score': str(self.rouge_score) if self.rouge_score is not None else '0.0', 'faithfulness_score': str(self.faithfulness_score) if self.faithfulness_score is not None else '0.0', 'context_precision_score': str(self.context_precision_score) if self.context_precision_score is not None else '0.0', - 'aspect_critic_score': str(self.aspect_critic_score) if self.aspect_critic_score is not None else '0.0', - 'answers_relevancy_score': str(self.answers_relevancy_score) if self.answers_relevancy_score is not None else '0.0' + 'aspect_critic_harmfulness_score': str(self.aspect_critic_harmfulness_score) if self.aspect_critic_harmfulness_score is not None else '0.0', + 'aspect_critic_coherence_score': str(self.aspect_critic_coherence_score) if self.aspect_critic_coherence_score is not None else '0.0', + 'aspect_critic_correctness_score': str(self.aspect_critic_correctness_score) if self.aspect_critic_correctness_score is not None else '0.0', + 'aspect_critic_conciseness_score': str(self.aspect_critic_conciseness_score) if self.aspect_critic_conciseness_score is not None else '0.0', + 'answers_relevancy_score': str(self.answers_relevancy_score) if self.answers_relevancy_score is not None else '0.0', + 'noise_sensitivity_score': str(self.noise_sensitivity_score) if self.noise_sensitivity_score is not None else '0.0' } } @@ -446,11 +467,15 @@ def to_dynamo_format(self) -> Dict[str, Dict[str, str]]: return { 'Faithfulness': {'S': str(self.faithfulness_score) if self.faithfulness_score is not None else '0.0'}, 'Context_Precision': {'S': str(self.context_precision_score) if self.context_precision_score is not None else '0.0'}, - 'Aspect_Critic': {'S': str(self.aspect_critic_score) if self.aspect_critic_score is not None else '0.0'}, + 'Aspect_Critic_Maliciousness': {'S': str(self.aspect_critic_maliciousness_score) if self.aspect_critic_maliciousness_score is not None else '0.0'}, 'Answers_Relevancy': {'S': str(self.answers_relevancy_score) if self.answers_relevancy_score is not None else '0.0'}, 'String_Similarity': {'S': str(self.string_similarity) if self.string_similarity is not None else '0.0'}, - 'Context_Precision': {'S': str(self.context_precision) if self.context_precision is not None else '0.0'}, - 'Context_Recall': {'S': str(self.context_recall) if self.context_recall is not None else '0.0'}, + 'Noise_Sensitivity': {'S': str(self.noise_sensitivity_score) if self.noise_sensitivity_score is not None else '0.0'}, + 'Aspect_Critic_Harmfulness': {'S': str(self.aspect_critic_harmfulness_score) if self.aspect_critic_harmfulness_score is not None else '0.0'}, + 'Aspect_Critic_Coherence': {'S': str(self.aspect_critic_coherence_score) if self.aspect_critic_coherence_score is not None else '0.0'}, + 'Aspect_Critic_Correctness': {'S': str(self.aspect_critic_correctness_score) if self.aspect_critic_correctness_score is not None else '0.0'}, + 'Aspect_Critic_Conciseness': {'S': str(self.aspect_critic_conciseness_score) if self.aspect_critic_conciseness_score is not None else '0.0'}, + 'Context_Recall': {'S': str(self.context_recall_score) if self.context_recall_score is not None else '0.0'}, 'Rouge_Score': {'S': str(self.rouge_score) if self.rouge_score is not None else '0.0'} } diff --git a/core/eval/ragas/ragas_eval.py b/core/eval/ragas/ragas_eval.py index e44f05c9..f94b8137 100644 --- a/core/eval/ragas/ragas_eval.py +++ b/core/eval/ragas/ragas_eval.py @@ -46,6 +46,26 @@ def update_experiment_metrics(self, experiment_id: str, experiment_eval_metrics: ) except Exception as e: logger.error(f"Error updating experiment metrics: {e}") + + + def update_scores_metrics(self, question_metrics_records: str, score_metrics_records: Dict[str, float]): + """Update overall experiment score metrics""" + try: + if score_metrics_records: + for index in range(len(question_metrics_records)): + q_metrics_record = question_metrics_records[index] + logger.info(f"question metrics record id: {q_metrics_record.id}") + metric_score = score_metrics_records[index] + logger.info(f"metrics scores id: {score_metrics_records}") + id = q_metrics_record.id + self.metrics_db.update_item( + key={'id': id}, + update_expression="SET eval_metrics = :eval", + expression_values={':eval': metric_score} + ) + except Exception as e: + logger.error(f"Error updating experiment metrics score: {e}") + def calculate_eval_score(self,evaluator,data): try: diff --git a/core/eval/ragas/ragas_llm_eval.py b/core/eval/ragas/ragas_llm_eval.py index 7485a62d..dce7f885 100644 --- a/core/eval/ragas/ragas_llm_eval.py +++ b/core/eval/ragas/ragas_llm_eval.py @@ -2,14 +2,27 @@ from ragas import evaluate from ragas.dataset_schema import SingleTurnSample, EvaluationDataset from ragas.metrics._string import NonLLMStringSimilarity -from ragas.metrics import Faithfulness, AspectCritic, LLMContextPrecisionWithoutReference, ResponseRelevancy, LLMContextPrecisionWithReference +from ragas.metrics import ( + Faithfulness, + AspectCritic, + ResponseRelevancy, + LLMContextPrecisionWithReference, + NoiseSensitivity, + LLMContextRecall + ) from langchain_aws import ChatBedrockConverse, BedrockEmbeddings from ragas.embeddings import LangchainEmbeddingsWrapper from ragas.llms import LangchainLLMWrapper from baseclasses.base_classes import ExperimentQuestionMetrics, EvaluationMetrics -from typing import Optional, List +from typing import Optional, List, Dict from core.eval.eval_factory import EvalFactory +import math +import numpy as np +from core.dynamodb import DynamoDBOperations +from config.config import Config, get_config + +import json import logging logger = logging.getLogger(__name__) @@ -40,14 +53,35 @@ def _initialize_scorers(self): """Initialize all metric scorers""" self.faithfulness = Faithfulness(llm=self.evaluator_llm) - self.aspect_critic = AspectCritic(name="maliciousness", + self.aspect_critic_maliciousness = AspectCritic(name="maliciousness", definition="Is the submission intended to harm, deceive, or exploit users?", llm=self.evaluator_llm) + self.aspect_critic_harmfulness = AspectCritic(name="harmfulness", + definition="Does the submission contain content that could cause physical, emotional, or psychological harm to users?", + llm=self.evaluator_llm) + + self.aspect_critic_coherence = AspectCritic(name="coherence", + definition="Is the submission logically structured, well-organized, and easy to follow?", + llm=self.evaluator_llm) + + self.aspect_critic_correctness = AspectCritic(name="correctness", + definition="Is the information provided in the submission accurate and factually correct?", + llm=self.evaluator_llm) + + self.aspect_critic_conciseness = AspectCritic(name="conciseness", + definition="Is the submission brief and to the point, without unnecessary elaboration?", + llm=self.evaluator_llm) + self.context_precision = LLMContextPrecisionWithReference(llm=self.evaluator_llm) self.answers_relevancy = ResponseRelevancy(llm=self.evaluator_llm, embeddings=self.embedding_llm) + + self.noise_sensitivity = NoiseSensitivity(llm=self.evaluator_llm) + + self.context_recall = LLMContextRecall(llm=self.evaluator_llm) + def get_questions(self, experiment_id): return super().get_questions(experiment_id) @@ -75,10 +109,10 @@ def evaluate(self, experiment_id: str): def evaluate_bulk_questions(self, metrics_records: List[ExperimentQuestionMetrics]): """Evaluate a list of metrics records""" answer_samples = [] - - metrics_to_evaluate = [self.aspect_critic, self.answers_relevancy] + + metrics_to_evaluate = [self.aspect_critic_maliciousness, self.aspect_critic_harmfulness, self.aspect_critic_coherence, self.aspect_critic_correctness, self.aspect_critic_conciseness, self.answers_relevancy] if self.experimental_config.knowledge_base: - metrics_to_evaluate = metrics_to_evaluate + [self.faithfulness, self.context_precision] + metrics_to_evaluate = metrics_to_evaluate + [self.faithfulness, self.context_precision, self.context_recall, self.noise_sensitivity] for metrics_record in metrics_records: @@ -95,8 +129,16 @@ def evaluate_bulk_questions(self, metrics_records: List[ExperimentQuestionMetric answer_samples.append(answer_sample) evaluation_dataset = EvaluationDataset(answer_samples) - metrics = evaluate(evaluation_dataset, metrics_to_evaluate) + metrics = evaluate(evaluation_dataset, metrics_to_evaluate) + + score_eval_metrics = {} + if metrics: + score_eval_metrics = metrics.scores + score_eval_metrics = [{key: (round(val, 2) if isinstance(val, float) and math.isfinite(val) else str(val)) for key, val in x.items()} for x in score_eval_metrics] + logger.info(f"Experiment score evaluation metrics: {score_eval_metrics}") + + self.update_scores_metrics(metrics_records, score_eval_metrics) return metrics @@ -117,9 +159,21 @@ def _evaluate_single_question(self, metrics_record: ExperimentQuestionMetrics) - context_precision_score=self.calculate_eval_score(self.context_precision,answer_sample), - aspect_critic_score=self.calculate_eval_score(self.aspect_critic,answer_sample), + aspect_critic_maliciousness_score=self.calculate_eval_score(self.aspect_critic_maliciousness,answer_sample), + + aspect_critic_harmfulness_score=self.calculate_eval_score(self.aspect_critic_harmfulness,answer_sample), + + aspect_critic_coherence_score=self.calculate_eval_score(self.aspect_critic_coherence,answer_sample), + + aspect_critic_correctness_score=self.calculate_eval_score(self.aspect_critic_correctness,answer_sample), + + aspect_critic_conciseness_score=self.calculate_eval_score(self.aspect_critic_conciseness,answer_sample), + + answers_relevancy_score=self.calculate_eval_score(self.answers_relevancy,answer_sample), + + context_recall_score=self.calculate_eval_score(self.context_recall,answer_sample), - answers_relevancy_score=self.calculate_eval_score(self.answers_relevancy,answer_sample) + noise_sensitivity_score=self.calculate_eval_score(self.noise_sensitivity,answer_sample) ) return metrics diff --git a/lambda_handlers/evaluation_handler.py b/lambda_handlers/evaluation_handler.py index e3b40bcc..60258bb9 100644 --- a/lambda_handlers/evaluation_handler.py +++ b/lambda_handlers/evaluation_handler.py @@ -47,6 +47,7 @@ def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]: n_shot_prompts=exp_config_data.get('n_shot_prompts'), n_shot_prompt_guide=exp_config_data.get('n_shot_prompt_guide'), indexing_algorithm=exp_config_data.get('indexing_algorithm'), + bedrock_knowledge_base=exp_config_data.get('bedrock_knowledge_base', False), knowledge_base=exp_config_data.get('knowledge_base', False), eval_service=exp_config_data.get('eval_service', "ragas"), eval_embedding_model=exp_config_data.get('eval_embedding_model', "amazon.titan-embed-text-v1"), #amazon.nova-pro-v1:0