diff --git a/app/seed_data.py b/app/seed_data.py index e72445a..fa38654 100644 --- a/app/seed_data.py +++ b/app/seed_data.py @@ -18,6 +18,7 @@ "bedrock_anthropic.claude-3-5-sonnet-20240620-v1:0": 5, "bedrock_us.anthropic.claude-3-7-sonnet-20250219-v1:0": 5, "bedrock_us.anthropic.claude-3-5-haiku-20241022-v1:0": 5, + "bedrock_us.anthropic.claude-sonnet-4-20250514-v1:0": 5, "bedrock_cohere.command-r-plus-v1:0": 25, "bedrock_cohere.command-r-v1:0": 14, "bedrock_us.meta.llama3-2-1b-instruct-v1:0": 14, diff --git a/core/inference/bedrock/bedrock_inferencer.py b/core/inference/bedrock/bedrock_inferencer.py index 248afd6..f1b841f 100644 --- a/core/inference/bedrock/bedrock_inferencer.py +++ b/core/inference/bedrock/bedrock_inferencer.py @@ -152,6 +152,7 @@ def _extract_response(self, response: Dict) -> str: "anthropic.claude-3-5-sonnet-20240620-v1:0", "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "us.anthropic.claude-3-5-haiku-20241022-v1:0", + "us.anthropic.claude-sonnet-4-20250514-v1:0", "amazon.titan-text-express-v1", "amazon.titan-text-lite-v1", "us.amazon.nova-lite-v1:0", diff --git a/ui/app/components/DownloadResultsButton.vue b/ui/app/components/DownloadResultsButton.vue index 57cfed7..cdfdb50 100644 --- a/ui/app/components/DownloadResultsButton.vue +++ b/ui/app/components/DownloadResultsButton.vue @@ -28,25 +28,26 @@ const downloadResults = () => { delete item["guardrail_input_assessment"]; delete item["guardrail_context_assessment"]; delete item["guardrail_output_assessment"]; - const { id, ...rest } = item; + const { id, question, gt_answer, generated_answer, ...rest } = item; return { + ...{"Question": item.question, "Ground Truth": item.gt_answer, "Generated Answer": item.generated_answer}, ...rest, ...assessments, }; } else { - const results = { + const baseResults = { "ID": item.id, "Status": item.experiment_status, "Inferencing Model": item.config.retrieval_model, "Estimated Cost": item.cost || (item.cost === 0 ? 0 : "NA"), - "Faithfulness": item.eval_metrics?.M?.faithfulness_score ||item.eval_metrics?.faithfulness_score || (item.eval_metrics?.M?.faithfulness_score === 0 ? 0 : "-"), + "Faithfulness": item.eval_metrics?.M?.faithfulness_score ||item.eval_metrics?.faithfulness_score || (item.eval_metrics?.M?.faithfulness_score === 0 ? 0 : "NA"), "Context Precision": - item.eval_metrics?.M?.context_precision_score || item.eval_metrics?.context_precision_score || (item.eval_metrics?.M?.context_precision_score === 0 ? 0 : "-"), + item.eval_metrics?.M?.context_precision_score || item.eval_metrics?.context_precision_score || (item.eval_metrics?.M?.context_precision_score === 0 ? 0 : "NA"), "Maliciousness": - item.eval_metrics?.M?.aspect_critic_score ||item.eval_metrics?.aspect_critic_score || (item.eval_metrics?.M?.aspect_critic_score === 0 ? 0 : "-"), + item.eval_metrics?.M?.aspect_critic_score ||item.eval_metrics?.aspect_critic_score || (item.eval_metrics?.M?.aspect_critic_score === 0 ? 0 : "NA"), "Answer Relevancy": - item.eval_metrics?.M?.answers_relevancy_score ||item.eval_metrics?.answers_relevancy_score || (item.eval_metrics?.M?.answers_relevancy_score === 0 ? 0 : "-"), - "Duration": item.total_time || (item.total_time === 0 ? 0 : "-"), + item.eval_metrics?.M?.answers_relevancy_score ||item.eval_metrics?.answers_relevancy_score || (item.eval_metrics?.M?.answers_relevancy_score === 0 ? 0 : "NA"), + "Duration": item.total_time || (item.total_time === 0 ? 0 : "NA"), "Embedding Model": item.config.embedding_model || "NA", "Evaluation Service": item.config.eval_service, "Evaluation Embedding Model": @@ -60,14 +61,30 @@ const downloadResults = () => { "Reranking Model": item.config.rerank_model_id || "NA", "Guardrail": item.config?.guardrail_name || "NA", "Bedrock KB Name": item.config?.kb_name || "NA", - "KNN": item.config?.knn_num || (item.config?.knn_num === 0 ? 0 : "NA"), + // "KNN": item.config?.knn_num === 'nan' ? "NA" : item.config?.knn_num || (item.config?.knn_num === 0 ? 0 : "NA"), + "KNN": item.config.knn_num ? item.config.knn_num : "NA", "N Shot Prompts": item.config.n_shot_prompts || (item.config.n_shot_prompts === 0 ? 0 : "NA"), "Expert Evaluation Scores": item.scores || (item.scores === 0 ? 0 : "NA"), - } + }; + + +if (item.config.eval_service === 'deepeval') { + return { + + ...baseResults, + "Context Recall": item.eval_metrics?.M?.context_recall_score || item.eval_metrics?.context_recall_score || (item.eval_metrics?.M?.context_recall_score === 0 ? 0 : "NA"), + "Context Relevancy": item.eval_metrics?.M?.context_relevancy || item.eval_metrics?.context_relevancy || (item.eval_metrics?.M?.context_relevancy === 0 ? 0 : "NA"), + "Hallucination": item.eval_metrics?.M?.hallucination || item.eval_metrics?.hallucination || (item.eval_metrics?.M?.hallucination === 0 ? 0 : "NA"), + + }; +} else if (item.config.eval_service === 'ragas') { + return baseResults; +} +return baseResults; + +} +}); - return results; - } - }); if (!props.questionMetrics) { stringifyData.sort((a, b) => a.ID.localeCompare(b.ID)); } diff --git a/ui/app/components/ModelSelect.vue b/ui/app/components/ModelSelect.vue index 316a46a..1b3e3f2 100644 --- a/ui/app/components/ModelSelect.vue +++ b/ui/app/components/ModelSelect.vue @@ -13,7 +13,7 @@ const items = computed(() => { return meta.indexingStrategy.embeddingService } if (props.model === "retrieval") { - return props.region === 'us-west-2'? meta.retrievalStrategy.llmService.filter((item => !item.label.includes('Scout'))) : meta.retrievalStrategy.llmService; + return props.region === 'us-west-2'? meta.retrievalStrategy.llmService.filter((item: any) => !item?.label.includes('Scout')) : meta.retrievalStrategy.llmService; } return []; }); diff --git a/ui/app/components/Project/Create/EvalStrategyStep.vue b/ui/app/components/Project/Create/EvalStrategyStep.vue index 1155131..d886f48 100644 --- a/ui/app/components/Project/Create/EvalStrategyStep.vue +++ b/ui/app/components/Project/Create/EvalStrategyStep.vue @@ -46,7 +46,6 @@ const { mutateAsync: getGuardrailsList, isPending: isFetchingGuardrailsList } = useMutation({ mutationFn: async () => { const response = await useGuardrailsList(props.region); - console.log(response) guardrailsList.value = response?.map((item) => { return { label: item.name + ' - ' + item.version, @@ -111,7 +110,8 @@ onMounted(() => { > diff --git a/ui/app/components/Project/Create/Form.vue b/ui/app/components/Project/Create/Form.vue index 4f1ab3c..807de75 100644 --- a/ui/app/components/Project/Create/Form.vue +++ b/ui/app/components/Project/Create/Form.vue @@ -64,17 +64,19 @@ const nextStep = () => { }), vector_dimension: state.indexing?.vector_dimension || [], indexing_algorithm: state.indexing?.indexing_algorithm || '', - embedding: state.indexing?.embedding?.map((pc) => { + embedding: state?.indexing?.embedding.filter((pc) => { + return pc?.model && pc?.service && pc?.label + })? state.indexing?.embedding?.map((pc) => { return { - model: pc?.value, + model: pc?.model, service: pc?.service, label: pc?.label } - }) || [{ + }) : [{ "model": "", "service": "", "label": "" - }] + }] }, retrieval: { n_shot_prompts: state.retrieval?.n_shot_prompts, diff --git a/ui/app/components/Project/Create/RetrievalStrategyStep.vue b/ui/app/components/Project/Create/RetrievalStrategyStep.vue index dbe6ab0..e0075ac 100644 --- a/ui/app/components/Project/Create/RetrievalStrategyStep.vue +++ b/ui/app/components/Project/Create/RetrievalStrategyStep.vue @@ -61,7 +61,6 @@ const tooltip = ref('') const fieldName = ref('') const handleTooltip = (tooltipInfo: {tooltip: string, fieldName: string}) => { - console.log(tooltipInfo) tooltip.value = tooltipInfo.tooltip fieldName.value = tooltipInfo.fieldName } @@ -139,9 +138,6 @@ const handleTooltip = (tooltipInfo: {tooltip: string, fieldName: string}) => { - >({ const toast = useToast() async function onSubmit(event: FormSubmitEvent) { toast.add({ title: 'Success', description: 'The form has been submitted.', color: 'success' }) - console.log(event.data) mutate(event.data) } diff --git a/ui/app/components/Project/Experiment/List.vue b/ui/app/components/Project/Experiment/List.vue index d080077..f8ebb4f 100644 --- a/ui/app/components/Project/Experiment/List.vue +++ b/ui/app/components/Project/Experiment/List.vue @@ -24,6 +24,8 @@ watchEffect(() => { } }) +const evalService = computed(() => props.experiments[0]?.config?.eval_service || ''); + const columns = ref[]>([ { id: 'select', @@ -193,31 +195,43 @@ const columns = ref[]>([ label: 'Faithfulness', sortingFn: (rowA, rowB) => { const getScore = (row: any) => { + let score; if ("faithfulness_score" in row.original.eval_metrics) { - return row.original.eval_metrics.faithfulness_score ?? 0; - } - if ("M" in row.original.eval_metrics) { - return row.original.eval_metrics.M?.faithfulness_score ?? 0; + score = row.original.eval_metrics.faithfulness_score; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.faithfulness_score; + } else { + score = 0; } - return 0; + + // Handle 'nan' string case + if (score === 'nan') return null; + // Convert to number, if NaN then return null + const numScore = Number(score); + return isNaN(numScore) ? null : numScore; }; const a = getScore(rowA); const b = getScore(rowB); - return Number(a) - Number(b); + + // Handle null values (which includes 'nan' and NaN) by placing them at the end + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + + return a - b; }, cell: ({ row }) => { - if(!row.original.config.knowledge_base){ - return "NA" - - } + if(!row.original.config.knowledge_base){ + return "NA" + } if ("faithfulness_score" in row.original.eval_metrics) { - return row.original.eval_metrics.faithfulness_score ? parseFloat(row.original.eval_metrics.faithfulness_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.faithfulness_score ? parseFloat(row.original.eval_metrics.faithfulness_score.toString()).toFixed(2) : "NA" } if ("M" in row.original.eval_metrics) { - return row.original.eval_metrics.M?.faithfulness_score ? parseFloat(row.original.eval_metrics.M.faithfulness_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.M?.faithfulness_score ? parseFloat(row.original.eval_metrics.M.faithfulness_score.toString()).toFixed(2) : "NA" } - return "-" + return "NA" } }, { @@ -243,30 +257,44 @@ const columns = ref[]>([ enableHiding: true, label: 'Context Precision', sortingFn: (rowA, rowB) => { - if ("M" in rowA.original.eval_metrics) { - const a = rowA.original.eval_metrics?.M?.context_precision_score; - const b = rowB.original.eval_metrics?.M?.context_precision_score; - return Number(a) - Number(b); - } - if ("context_precision_score" in rowA.original.eval_metrics) { - const a = rowA.original.eval_metrics?.context_precision_score; - const b = rowB.original.eval_metrics?.context_precision_score; - return Number(a) - Number(b); - } - return 0; + const getScore = (row: any) => { + let score; + if ("context_precision_score" in row.original.eval_metrics) { + score = row.original.eval_metrics.context_precision_score; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.context_precision_score; + } else { + score = 0; + } + + // Handle 'nan' string case + if (score === 'nan') return null; + // Convert to number, if NaN then return null + const numScore = Number(score); + return isNaN(numScore) ? null : numScore; + }; + + const a = getScore(rowA); + const b = getScore(rowB); + + // Handle null values (which includes 'nan' and NaN) by placing them at the end + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + + return a - b; }, cell: ({ row }) => { - if(!row.original.config.knowledge_base){ - return "NA" - - } + if(!row.original.config.knowledge_base){ + return "NA" + } if ("context_precision_score" in row.original.eval_metrics) { - return row.original.eval_metrics.context_precision_score ? parseFloat(row.original.eval_metrics.context_precision_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.context_precision_score ? parseFloat(row.original.eval_metrics.context_precision_score.toString()).toFixed(2) : "NA" } if ("M" in row.original.eval_metrics) { - return row.original.eval_metrics.M?.context_precision_score ? parseFloat(row.original.eval_metrics.M.context_precision_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.M?.context_precision_score ? parseFloat(row.original.eval_metrics.M.context_precision_score.toString()).toFixed(2) : "NA" } - return "-" + return "NA" } }, { @@ -293,33 +321,193 @@ const columns = ref[]>([ enableHiding: true, sortingFn: (rowA, rowB) => { const getScore = (row: any) => { - let score = 0; - if ("M" in row.original.eval_metrics) { - score = row.original.eval_metrics?.M?.aspect_critic_score ?? 0; - } else if ("aspect_critic_score" in row.original.eval_metrics) { - score = row.original.eval_metrics?.aspect_critic_score ?? 0; + let score; + if ("aspect_critic_score" in row.original.eval_metrics) { + score = row.original.eval_metrics.aspect_critic_score; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.aspect_critic_score; + } else { + score = 0; } - return Number(score); + + // Handle 'nan' string case + if (score === 'nan') return null; + // Convert to number, if NaN then return null + const numScore = Number(score); + return isNaN(numScore) ? null : numScore; }; - + const a = getScore(rowA); const b = getScore(rowB); - // Handle NaN values by placing them at the end - if (isNaN(a) && isNaN(b)) return 0; - if (isNaN(a)) return 1; - if (isNaN(b)) return -1; + // Handle null values (which includes 'nan' and NaN) by placing them at the end + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; return a - b; }, cell: ({ row }) => { if ("aspect_critic_score" in row.original.eval_metrics) { - return row.original.eval_metrics.aspect_critic_score ? parseFloat(row.original.eval_metrics.aspect_critic_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.aspect_critic_score ? parseFloat(row.original.eval_metrics.aspect_critic_score.toString()).toFixed(2) : "NA" } if ("M" in row.original.eval_metrics) { - return row.original.eval_metrics.M?.aspect_critic_score ? parseFloat(row.original.eval_metrics.M.aspect_critic_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.M?.aspect_critic_score ? parseFloat(row.original.eval_metrics.M.aspect_critic_score.toString()).toFixed(2) : "NA" + } + return "NA" + } + }, + { + header: ({ column }) => { + if (evalService.value !== 'deepeval') return null; + const isSorted = column.getIsSorted(); + return h('div', { class: 'flex items-center justify-between' }, [ + h(UButton, { + color: "neutral", + variant: "ghost", + label: "Context Relevancy", + trailingIcon: isSorted + ? isSorted === "asc" + ? "i-lsicon:triangle-up-filled" + : "i-lsicon:triangle-down-filled" + : "i-lsicon:triangle-down-outline", + class: "-mx-2.5 focus:font-bold hover:font-bold hover:bg-white cursor-pointer", + onClick: () => column.toggleSorting(column.getIsSorted() === "asc"), + }), + h('div', { class: 'h-5 w-[2px] bg-gray-200 dark:bg-gray-700 ml-2' }) + ]); + }, + accessorKey: "eval_metrics.context_relevancy", + enableHiding: true, + label: 'Context Relevancy', + sortingFn: (rowA, rowB) => { + const getScore = (row: any) => { + let score; + if ("context_relevancy" in row.original.eval_metrics) { + score = row.original.eval_metrics.context_relevancy; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.context_relevancy; + } else { + score = 0; + } + if (score === 'nan') return null; + const num = Number(score); + return isNaN(num) ? null : num; + }; + + const a = getScore(rowA); + const b = getScore(rowB); + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + return a - b; + }, + cell: ({ row }) => { + if (row.original.config?.eval_service !== 'deepeval') return null; + const score = row.original.eval_metrics?.context_relevancy ?? row.original.eval_metrics?.M?.context_relevancy; + return score && score !== 'nan' ? parseFloat(score.toString()).toFixed(2) : "NA"; + } +}, +{ + header: ({ column }) => { + if (evalService.value !== 'deepeval') return null; + const isSorted = column.getIsSorted(); + return h('div', { class: 'flex items-center justify-between' }, [ + h(UButton, { + color: "neutral", + variant: "ghost", + label: "Hallucination", + trailingIcon: isSorted + ? isSorted === "asc" + ? "i-lsicon:triangle-up-filled" + : "i-lsicon:triangle-down-filled" + : "i-lsicon:triangle-down-outline", + class: "-mx-2.5 focus:font-bold hover:font-bold hover:bg-white cursor-pointer", + onClick: () => column.toggleSorting(column.getIsSorted() === "asc"), + }), + h('div', { class: 'h-5 w-[2px] bg-gray-200 dark:bg-gray-700 ml-2' }) + ]); + }, + accessorKey: "eval_metrics.hallucination", + enableHiding: true, + label: 'Hallucination', + sortingFn: (rowA, rowB) => { + const getScore = (row: any) => { + let score; + if ("hallucination" in row.original.eval_metrics) { + score = row.original.eval_metrics.hallucination; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.hallucination; + } else { + score = 0; + } + if (score === 'nan') return null; + const num = Number(score); + return isNaN(num) ? null : num; + }; + + const a = getScore(rowA); + const b = getScore(rowB); + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + return a - b; + }, + cell: ({ row }) => { + if (row.original.config?.eval_service !== 'deepeval') return null; + const score = row.original.eval_metrics?.hallucination ?? row.original.eval_metrics?.M?.hallucination; + return score && score !== 'nan' ? parseFloat(score.toString()).toFixed(2) : "NA"; + } +}, +{ + header: ({ column }) => { + if (evalService.value !== 'deepeval') return null; + const isSorted = column.getIsSorted(); + return h('div', { class: 'flex items-center justify-between' }, [ + h(UButton, { + color: "neutral", + variant: "ghost", + label: "Context Recall", + trailingIcon: isSorted + ? isSorted === "asc" + ? "i-lsicon:triangle-up-filled" + : "i-lsicon:triangle-down-filled" + : "i-lsicon:triangle-down-outline", + class: "-mx-2.5 focus:font-bold hover:font-bold hover:bg-white cursor-pointer", + onClick: () => column.toggleSorting(column.getIsSorted() === "asc"), + }), + h('div', { class: 'h-5 w-[2px] bg-gray-200 dark:bg-gray-700 ml-2' }) + ]); + }, + accessorKey: "eval_metrics.context_recall_score", + enableHiding: true, + label: 'Context Recall', + sortingFn: (rowA, rowB) => { + const getScore = (row: any) => { + let score; + if ("context_recall_score" in row.original.eval_metrics) { + score = row.original.eval_metrics.context_recall_score; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.context_recall_score; + } else { + score = 0; } - return "-" + if (score === 'nan') return null; + const num = Number(score); + return isNaN(num) ? null : num; + }; + + const a = getScore(rowA); + const b = getScore(rowB); + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + return a - b; + }, + cell: ({ row }) => { + if (row.original.config?.eval_service !== 'deepeval') return null; + const score = row.original.eval_metrics?.context_recall_score ?? row.original.eval_metrics?.M?.context_recall_score; + return score && score !== 'nan' ? parseFloat(score.toString()).toFixed(2) : "NA"; } }, { @@ -345,26 +533,41 @@ const columns = ref[]>([ enableHiding: true, label: 'Answer Relevancy', sortingFn: (rowA, rowB) => { - if ("M" in rowA.original.eval_metrics) { - const a = rowA.original.eval_metrics?.M?.answers_relevancy_score ?? 0; - const b = rowB.original.eval_metrics?.M?.answers_relevancy_score ?? 0; - return Number(a) - Number(b); - } - if ("answers_relevancy_score" in rowA.original.eval_metrics) { - const a = rowA.original.eval_metrics?.answers_relevancy_score ?? 0; - const b = rowB.original.eval_metrics?.answers_relevancy_score ?? 0; - return Number(a) - Number(b); - } - return 0; + const getScore = (row: any) => { + let score; + if ("answers_relevancy_score" in row.original.eval_metrics) { + score = row.original.eval_metrics.answers_relevancy_score; + } else if ("M" in row.original.eval_metrics) { + score = row.original.eval_metrics.M?.answers_relevancy_score; + } else { + score = 0; + } + + // Handle 'nan' string case + if (score === 'nan') return null; + // Convert to number, if NaN then return null + const numScore = Number(score); + return isNaN(numScore) ? null : numScore; + }; + + const a = getScore(rowA); + const b = getScore(rowB); + + // Handle null values (which includes 'nan' and NaN) by placing them at the end + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + + return a - b; }, cell: ({ row }) => { if ("answers_relevancy_score" in row.original.eval_metrics) { - return row.original.eval_metrics.answers_relevancy_score ? parseFloat(row.original.eval_metrics.answers_relevancy_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.answers_relevancy_score ? parseFloat(row.original.eval_metrics.answers_relevancy_score.toString()).toFixed(2) : "NA" } if ("M" in row.original.eval_metrics) { - return row.original.eval_metrics.M?.answers_relevancy_score ? parseFloat(row.original.eval_metrics.M.answers_relevancy_score.toString()).toFixed(2) : "-" + return row.original.eval_metrics.M?.answers_relevancy_score ? parseFloat(row.original.eval_metrics.M.answers_relevancy_score.toString()).toFixed(2) : "NA" } - return "-" + return "NA" } }, { @@ -849,6 +1052,9 @@ const columnVisibility = ref({ config_chunking_strategy: false, config_embedding_model: false, scores: false, + 'eval_metrics.context_recall_score': evalService.value === 'deepeval', + 'eval_metrics.context_relevancy': evalService.value === 'deepeval', + 'eval_metrics.hallucination': evalService.value === 'deepeval' }) diff --git a/ui/app/composables/shared.ts b/ui/app/composables/shared.ts index 93ffcc1..3bc79c6 100644 --- a/ui/app/composables/shared.ts +++ b/ui/app/composables/shared.ts @@ -391,6 +391,11 @@ export const useProjectCreateMeta = () => { value: "us.anthropic.claude-3-7-sonnet-20250219-v1:0", service: "bedrock", }, + { + label: "Anthropic/us.anthropic.claude-sonnet-4-20250514-v1:0", + value: "us.anthropic.claude-sonnet-4-20250514-v1:0", + service: "bedrock", + }, { type: "label", label: "SageMaker (Provisioned)", @@ -507,9 +512,13 @@ export const useProjectCreateMeta = () => { evalStrategy: { service: [ { - label: 'ragas', + label: 'Ragas', value: 'ragas', }, + { + label: 'Deep Eval', + value: 'deepeval', + }, { label: 'others - coming soon', value: 'others', @@ -727,7 +736,7 @@ export const useProjectUploadConfig = () => { ...(config.indexing?.chunking_strategy.includes('hierarchical') && config.indexing?.hierarchical_parent_chunk_size.length > 0 ? { hierarchical_parent_chunk_size: config.indexing?.hierarchical_parent_chunk_size} : {}), ...(config.indexing?.chunking_strategy.includes('hierarchical') && config.indexing?.hierarchical_child_chunk_size.length > 0 ? { hierarchical_child_chunk_size: config.indexing?.hierarchical_child_chunk_size} : {}), ...(config.indexing?.chunking_strategy.includes('hierarchical') && config.indexing?.hierarchical_chunk_overlap_percentage.length > 0 ? { hierarchical_chunk_overlap_percentage: config.indexing?.hierarchical_chunk_overlap_percentage} : {}), - embedding: config.indexing?.embedding?.map((pc: any) => { + embedding: config.indexing.embedding.length === 1 && !config.indexing.embedding[0].model ? [{"model": '', "service": '', "label": ''}] : config.indexing.embedding.map((pc: any) => { return useGetModelData("indexing", pc.model); }), }, diff --git a/ui/public/flotorch-hero.gif b/ui/public/flotorch-hero.gif index b796a54..0b81539 100644 Binary files a/ui/public/flotorch-hero.gif and b/ui/public/flotorch-hero.gif differ diff --git a/ui/shared/types/projects.type.ts b/ui/shared/types/projects.type.ts index cbfa615..55ae075 100644 --- a/ui/shared/types/projects.type.ts +++ b/ui/shared/types/projects.type.ts @@ -398,6 +398,9 @@ export interface EvalMetrics { context_precision_score: number; aspect_critic_score: number; answers_relevancy_score: number; + context_recall_score: number; + context_relevancy: number; + hallucination: number; } export interface EvalMetricsM {