Feature/integrating deepeval metics

app/seed_data.py

@@ -18,6 +18,7 @@
     "bedrock_anthropic.claude-3-5-sonnet-20240620-v1:0": 5,
     "bedrock_us.anthropic.claude-3-7-sonnet-20250219-v1:0": 5,
     "bedrock_us.anthropic.claude-3-5-haiku-20241022-v1:0": 5,
+    "bedrock_us.anthropic.claude-sonnet-4-20250514-v1:0": 5,
     "bedrock_cohere.command-r-plus-v1:0": 25,
     "bedrock_cohere.command-r-v1:0": 14,
     "bedrock_us.meta.llama3-2-1b-instruct-v1:0": 14,

core/inference/bedrock/bedrock_inferencer.py

@@ -152,6 +152,7 @@ def _extract_response(self, response: Dict) -> str:
               "anthropic.claude-3-5-sonnet-20240620-v1:0",
               "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
               "us.anthropic.claude-3-5-haiku-20241022-v1:0",
+              "us.anthropic.claude-sonnet-4-20250514-v1:0",
               "amazon.titan-text-express-v1",
               "amazon.titan-text-lite-v1",
               "us.amazon.nova-lite-v1:0",

ui/app/components/DownloadResultsButton.vue

@@ -28,25 +28,26 @@ const downloadResults = () => {
       delete item["guardrail_input_assessment"];
       delete item["guardrail_context_assessment"];
       delete item["guardrail_output_assessment"];
-      const { id, ...rest } = item;
+      const { id, question, gt_answer, generated_answer, ...rest } = item;
       return {
+        ...{"Question": item.question, "Ground Truth": item.gt_answer, "Generated Answer": item.generated_answer},
         ...rest,
         ...assessments,
       };
     } else {
-      const results = {
+      const baseResults = {
         "ID": item.id,
         "Status": item.experiment_status,
         "Inferencing Model": item.config.retrieval_model,
         "Estimated Cost": item.cost || (item.cost === 0 ? 0 : "NA"),
-        "Faithfulness": item.eval_metrics?.M?.faithfulness_score ||item.eval_metrics?.faithfulness_score || (item.eval_metrics?.M?.faithfulness_score === 0 ? 0 : "-"),
+        "Faithfulness": item.eval_metrics?.M?.faithfulness_score ||item.eval_metrics?.faithfulness_score || (item.eval_metrics?.M?.faithfulness_score === 0 ? 0 : "NA"),
         "Context Precision":
-        item.eval_metrics?.M?.context_precision_score || item.eval_metrics?.context_precision_score || (item.eval_metrics?.M?.context_precision_score === 0 ? 0 : "-"),
+        item.eval_metrics?.M?.context_precision_score || item.eval_metrics?.context_precision_score || (item.eval_metrics?.M?.context_precision_score === 0 ? 0 : "NA"),
         "Maliciousness":
-          item.eval_metrics?.M?.aspect_critic_score ||item.eval_metrics?.aspect_critic_score || (item.eval_metrics?.M?.aspect_critic_score === 0 ? 0 : "-"),
+          item.eval_metrics?.M?.aspect_critic_score ||item.eval_metrics?.aspect_critic_score || (item.eval_metrics?.M?.aspect_critic_score === 0 ? 0 : "NA"),
         "Answer Relevancy":
-          item.eval_metrics?.M?.answers_relevancy_score ||item.eval_metrics?.answers_relevancy_score || (item.eval_metrics?.M?.answers_relevancy_score === 0 ? 0 : "-"),
-        "Duration": item.total_time || (item.total_time === 0 ? 0 : "-"),
+          item.eval_metrics?.M?.answers_relevancy_score ||item.eval_metrics?.answers_relevancy_score || (item.eval_metrics?.M?.answers_relevancy_score === 0 ? 0 : "NA"),
+        "Duration": item.total_time || (item.total_time === 0 ? 0 : "NA"),
         "Embedding Model": item.config.embedding_model || "NA",
         "Evaluation Service": item.config.eval_service,
         "Evaluation Embedding Model":
@@ -60,14 +61,30 @@ const downloadResults = () => {
         "Reranking Model": item.config.rerank_model_id || "NA",
         "Guardrail": item.config?.guardrail_name || "NA",
         "Bedrock KB Name": item.config?.kb_name || "NA",
-        "KNN": item.config?.knn_num || (item.config?.knn_num === 0 ? 0 : "NA"),
+        // "KNN": item.config?.knn_num === 'nan' ? "NA" : item.config?.knn_num || (item.config?.knn_num === 0 ? 0 : "NA"),
+        "KNN": item.config.knn_num ? item.config.knn_num : "NA",
         "N Shot Prompts": item.config.n_shot_prompts || (item.config.n_shot_prompts === 0 ? 0 : "NA"),
         "Expert Evaluation Scores": item.scores || (item.scores === 0 ? 0 : "NA"),
-      }
+      };
+
+
+if (item.config.eval_service === 'deepeval') {
+  return {
+
+          ...baseResults,
+         "Context Recall": item.eval_metrics?.M?.context_recall_score || item.eval_metrics?.context_recall_score || (item.eval_metrics?.M?.context_recall_score === 0 ? 0 : "NA"),
+         "Context Relevancy": item.eval_metrics?.M?.context_relevancy || item.eval_metrics?.context_relevancy || (item.eval_metrics?.M?.context_relevancy === 0 ? 0 : "NA"),
+         "Hallucination": item.eval_metrics?.M?.hallucination || item.eval_metrics?.hallucination || (item.eval_metrics?.M?.hallucination === 0 ? 0 : "NA"),
+
+  };
+} else if (item.config.eval_service === 'ragas') {
+  return baseResults;
+}
+return baseResults;
+
+}
+});
 
-      return results;
-    }
-  });
   if (!props.questionMetrics) {
     stringifyData.sort((a, b) => a.ID.localeCompare(b.ID));
   }

ui/app/components/ModelSelect.vue

@@ -13,7 +13,7 @@ const items = computed(() => {
     return meta.indexingStrategy.embeddingService
   }
   if (props.model === "retrieval") {
-    return props.region === 'us-west-2'? meta.retrievalStrategy.llmService.filter((item => !item.label.includes('Scout'))) : meta.retrievalStrategy.llmService;
+    return props.region === 'us-west-2'? meta.retrievalStrategy.llmService.filter((item: any) => !item?.label.includes('Scout')) : meta.retrievalStrategy.llmService;
   }
   return [];
 });

ui/app/components/Project/Create/EvalStrategyStep.vue

@@ -46,7 +46,6 @@ const { mutateAsync: getGuardrailsList, isPending: isFetchingGuardrailsList } =
   useMutation({
     mutationFn: async () => {
       const response = await useGuardrailsList(props.region);
-      console.log(response)
       guardrailsList.value = response?.map((item) => {
         return {
           label: item.name + ' - ' + item.version,
@@ -111,7 +110,8 @@ onMounted(() => {
         >
           <template #label="{ label }">
             <div class="flex items-center">
-              {{ label }}
+              {{ label }}<span class="italic"> - required</span>
+              <span class="w-[1px] h-3 ml-2 bg-gray-400"></span>
               <FieldTooltip @show-tooltip="handleTooltip" field-name="guardrails"/>
             </div>
           </template>

ui/app/components/Project/Create/Form.vue

@@ -64,17 +64,19 @@ const nextStep = () => {
         }),
         vector_dimension: state.indexing?.vector_dimension || [],
         indexing_algorithm: state.indexing?.indexing_algorithm || '',
-        embedding: state.indexing?.embedding?.map((pc) => {
+        embedding: state?.indexing?.embedding.filter((pc) => {
+          return pc?.model && pc?.service && pc?.label
+        })? state.indexing?.embedding?.map((pc) => {
           return {
-            model: pc?.value,
+            model: pc?.model,
             service: pc?.service,
             label: pc?.label
           }
-        }) || [{
+        }) : [{
                 "model": "",
                 "service": "",
                 "label": ""
-            }]
+            }] 
       },
       retrieval: {
         n_shot_prompts: state.retrieval?.n_shot_prompts,

ui/app/components/Project/Create/RetrievalStrategyStep.vue

@@ -61,7 +61,6 @@ const tooltip = ref('')
 const fieldName = ref('')
 
 const handleTooltip = (tooltipInfo: {tooltip: string, fieldName: string}) => {
-  console.log(tooltipInfo)
   tooltip.value = tooltipInfo.tooltip
   fieldName.value = tooltipInfo.fieldName
 }
@@ -139,9 +138,6 @@ const handleTooltip = (tooltipInfo: {tooltip: string, fieldName: string}) => {
           <FieldTooltip @show-tooltip="handleTooltip" field-name="retrieval"/>
         </div>
       </template>
-      <!-- <template #hint>
-        <FieldTooltip field-name="retrieval" />
-      </template> -->
     </UFormField>
     <UFormField name="temp_retrieval_llm"
       :label="`Inferencing Model Temperature ${state?.temp_retrieval_llm?.length === 0 || state?.temp_retrieval_llm === undefined ? '' : `(${state?.temp_retrieval_llm?.length})`}`"

ui/app/components/Project/Experiment/HumanEvaluation.vue

@@ -38,7 +38,6 @@ const state = reactive<Partial<HumanEval>>({
 const toast = useToast()
 async function onSubmit(event: FormSubmitEvent<HumanEval>) {
   toast.add({ title: 'Success', description: 'The form has been submitted.', color: 'success' })
-  console.log(event.data)
   mutate(event.data)
 }