From 98193e8c9a99929f9f2b815e6fb912a6ceb6ef7a Mon Sep 17 00:00:00 2001 From: guitavano Date: Tue, 7 Jul 2026 17:06:40 -0300 Subject: [PATCH] Add A/B test results tool with interactive UI Ports the deco.cx admin Experiments results screen into the MCP App as a new tool with an interactive dashboard. - `list_experiments`: entry-point tool that launches the UI, listing the site's experiments and resolving its production domains - `experiment_results`: fetches per-goal conversions, a daily timeseries, and the A/B statistics (participants, target sample size, probability each variant is best), computed server-side - `create_experiment`: creates a new experiment from the UI - `api/lib/ab-test.ts`: server-side port of the admin A/B math, with a jstat-free standard-normal CDF (Abramowitz & Stegun erf approximation) - React UI (recharts + shadcn): experiment list with a New dialog, results view with domain/period/filter controls, variant probability, timeseries and goal charts, and funnel/by-goal tables Co-Authored-By: Claude Opus 4.8 --- api/app.ts | 2 + api/lib/ab-test.ts | 100 +++ api/resources/experiments.ts | 23 + api/tools/experiments.ts | 432 ++++++++++++ api/tools/index.ts | 8 + web/router.tsx | 2 + web/tools/experiments/index.tsx | 1082 +++++++++++++++++++++++++++++++ 7 files changed, 1649 insertions(+) create mode 100644 api/lib/ab-test.ts create mode 100644 api/resources/experiments.ts create mode 100644 api/tools/experiments.ts create mode 100644 web/tools/experiments/index.tsx diff --git a/api/app.ts b/api/app.ts index 19aa01d..be10c45 100644 --- a/api/app.ts +++ b/api/app.ts @@ -1,5 +1,6 @@ import { withRuntime } from "@decocms/runtime"; import { createAssetsAppResource } from "./resources/assets.ts"; +import { createExperimentsAppResource } from "./resources/experiments.ts"; import { createLogsAppResource } from "./resources/logs.ts"; import { createMonitorAppResource } from "./resources/monitor.ts"; import { createReleasesAppResource } from "./resources/releases.ts"; @@ -100,6 +101,7 @@ export function createApp(opts: CreateAppOptions): Fetcher { tools, resources: [ createAssetsAppResource(getClientHTML), + createExperimentsAppResource(getClientHTML), createLogsAppResource(getClientHTML), createMonitorAppResource(getClientHTML), createReleasesAppResource(getClientHTML), diff --git a/api/lib/ab-test.ts b/api/lib/ab-test.ts new file mode 100644 index 0000000..e38d5e5 --- /dev/null +++ b/api/lib/ab-test.ts @@ -0,0 +1,100 @@ +/** + * A/B test statistics — server-side port of the deco.cx admin util. + * + * Reference: `deco-sites/admin/utils/statistics/abTest.ts` + * (components/spaces/siteEditor/extensions/CMS/views/Experiments/Experiments.tsx + * is the sole consumer over there, computing these numbers in the browser). + * + * The admin version depends on `jstat` purely for `jStat.normal.cdf`. Pulling + * the whole ~45KB library in just for one function is wasteful — especially + * here, where the client bundle is inlined into a single HTML resource. So we + * reimplement the standard-normal CDF with an Abramowitz & Stegun erf + * approximation (formula 7.1.26, |error| < 1.5e-7) and keep the rest of the math + * identical to the admin implementation. Everything runs inside the MCP tool so + * the UI only renders the results. + */ + +export interface Variant { + /** Number of successes (e.g. conversions for the selected goal). */ + successes: number; + /** Total participants (e.g. visitors that saw this variant). */ + total: number; +} + +// alpha = 0.05, one-sided +const Z_ALPHA = 1.644853; +// beta = 0.2, one-sided +const Z_BETA = 0.8416; +const MIN_SAMPLE_SIZE = 1000; + +/** + * Standard-normal cumulative distribution function. + * Drop-in replacement for `jStat.normal.cdf(x, mean, std)`. + */ +export function normalCdf(x: number, mean = 0, std = 1): number { + const z = (x - mean) / (std * Math.SQRT2); + // erf approximation — Abramowitz & Stegun 7.1.26 + const t = 1 / (1 + 0.3275911 * Math.abs(z)); + const y = + 1 - + ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * + t + + 0.254829592) * + t * + Math.exp(-z * z); + const erf = z >= 0 ? y : -y; + return 0.5 * (1 + erf); +} + +function proportion(a: Variant): number { + return a.successes / a.total; +} + +function standardError(a: Variant): number { + const p = proportion(a); + return Math.sqrt((p * (1 - p)) / a.total); +} + +/** + * Probability that a random sample from B is greater than one from A — + * i.e. the chance the test variant beats the default. + */ +export function pBetter(a: Variant, b: Variant): number { + const z = + -(proportion(b) - proportion(a)) / + Math.sqrt(standardError(a) ** 2 + standardError(b) ** 2); + const result = 1 - normalCdf(z, 0, 1); + return Number.isFinite(result) ? result : 0; +} + +/** + * Required sample size to reach significance. + * Based on Wiley Series in Probability and Statistics, Chapter 4. + * Returns `null` when there isn't enough data yet to establish a control. + */ +export function sampleSize(a: Variant, b: Variant, mde = 0.03): number | null { + // not enough data to establish a control for the sample size calculation + if (a.total < MIN_SAMPLE_SIZE || a.successes === 0) { + return null; + } + + const r = b.total / a.total; + + const p1 = proportion(a); + // use MDE if variant B's proportion would cause a sample size that is too large + const p2 = Math.max(p1 * (1 + mde), proportion(b)); + const p = (p1 + r * p2) / (r + 1); + + const numerator = + Z_ALPHA * Math.sqrt((r + 1) * p * (1 - p)) + + Z_BETA * Math.sqrt(r * p1 * (1 - p1) + p2 * (1 - p2)); + const denominator = p2 - p1; + const sampleSizeRaw = (numerator / denominator) ** 2 / r; + + const mul = + 1 + Math.sqrt(1 + (2 * (r + 1)) / (sampleSizeRaw * r * Math.abs(p1 - p2))); + + const size = (sampleSizeRaw * mul ** 2) / 4; + + return Math.ceil(size) * (1 + r); +} diff --git a/api/resources/experiments.ts b/api/resources/experiments.ts new file mode 100644 index 0000000..0ef25a0 --- /dev/null +++ b/api/resources/experiments.ts @@ -0,0 +1,23 @@ +import { createPublicResource } from "@decocms/runtime/tools"; +import { EXPERIMENTS_RESOURCE_URI } from "../tools/experiments.ts"; + +const RESOURCE_MIME_TYPE = "text/html;profile=mcp-app"; + +export const createExperimentsAppResource = ( + getClientHTML: () => Promise, +) => + createPublicResource({ + uri: EXPERIMENTS_RESOURCE_URI, + name: "A/B Test Results UI", + description: + "Interactive A/B test results dashboard: variant conversions, timeseries, and significance statistics for deco.cx experiments", + mimeType: RESOURCE_MIME_TYPE, + read: async () => { + const html = await getClientHTML(); + return { + uri: EXPERIMENTS_RESOURCE_URI, + mimeType: RESOURCE_MIME_TYPE, + text: html, + }; + }, + }); diff --git a/api/tools/experiments.ts b/api/tools/experiments.ts new file mode 100644 index 0000000..9c6889f --- /dev/null +++ b/api/tools/experiments.ts @@ -0,0 +1,432 @@ +import { createTool } from "@decocms/runtime/tools"; +import { z } from "zod"; +import { + pBetter, + sampleSize as sampleSizeOf, + type Variant, +} from "../lib/ab-test.ts"; +import { callAdmin, decodeJwtPayload, getConfig } from "../lib/admin.ts"; + +export const EXPERIMENTS_RESOURCE_URI = "ui://mcp-app/experiments"; + +const EXPERIMENTS_LIST_LOADER = "deco-sites/admin/loaders/experiments/list.ts"; +const EXPERIMENTS_CREATE_ACTION = + "deco-sites/admin/actions/experiments/create.ts"; +const SITES_DOMAINS_LOADER = "deco-sites/admin/loaders/sites/domains.ts"; +const ANALYTICS_AGGREGATE_LOADER = + "deco-sites/admin/loaders/analytics/aggregate.ts"; +const ANALYTICS_TIMESERIES_LOADER = + "deco-sites/admin/loaders/analytics/timeseries.ts"; + +// "visitors" is the implicit baseline goal — every stat is relative to it. +const VISITORS_GOAL = "visitors"; + +// ─── helpers ────────────────────────────────────────────────────────────────── + +async function invoke( + loader: string, + props: Record, + apiKey: string, +): Promise { + return (await callAdmin(loader, props, apiKey)) as T; +} + +function extractHostname(raw: string): string { + return raw.replace(/^https?:\/\//, "").replace(/\/$/, ""); +} + +// ─── shared schemas ───────────────────────────────────────────────────────── + +// Mirrors the periods offered by the admin Experiments view (custom is a range). +const periodPreset = z.enum(["day", "7d", "30d", "month", "6mo", "12mo"]); + +const dateRangeSchema = z.union([ + z.object({ type: z.literal("preset"), value: periodPreset }), + z.object({ + type: z.literal("custom"), + from: z.string().describe("YYYY-MM-DD"), + to: z.string().describe("YYYY-MM-DD"), + }), +]); + +const filtersSchema = z + .object({ + devices: z.array(z.string()).optional(), + browsers: z.array(z.string()).optional(), + os: z.array(z.string()).optional(), + }) + .optional(); + +// Matches the `experiments` table (see admin clients/supabase/types.ts) — every +// column except `id` is nullable, so keep the schema equally permissive: a +// stricter shape would make the runtime reject the whole result on a single +// null field, which surfaces to the UI as "no experiments". +const experimentSchema = z + .object({ + id: z.number(), + name: z.string().nullable().optional(), + status: z.string().nullable().optional(), + description: z.string().nullable().optional(), + startedAt: z.string().nullable().optional(), + endedAt: z.string().nullable().optional(), + site: z.string().nullable().optional(), + createdBy: z.string().nullable().optional(), + custom_goals: z.array(z.string()).nullable().optional(), + variants: z.unknown().optional(), + }) + .passthrough(); + +// ─── 1. Entry-point tool — launches the UI ──────────────────────────────────── + +export const listExperimentsTool = createTool({ + id: "list_experiments", + title: "A/B Test Results", + description: + "Open the A/B test results dashboard for the configured deco.cx site. Lists the site's experiments and resolves its production domains, then launches the interactive results UI.", + inputSchema: z.object({}), + outputSchema: z.object({ + sitename: z.string(), + hostname: z.string(), + domains: z.array(z.string()), + experiments: z.array(experimentSchema), + // Diagnostic: populated when the experiments loader itself fails, so the + // UI can distinguish "genuinely empty" from "the request errored". + error: z.string().optional(), + }), + _meta: { ui: { resourceUri: EXPERIMENTS_RESOURCE_URI } }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + execute: async (_input, ctx) => { + const { apiKey, site } = getConfig(ctx); + + type DomainEntry = { domain: string; production?: boolean }; + + let experiments: z.infer[] = []; + let error: string | undefined; + try { + experiments = await invoke[]>( + EXPERIMENTS_LIST_LOADER, + { site }, + apiKey, + ); + } catch (e) { + error = e instanceof Error ? e.message : String(e); + console.error(`[list_experiments] site=${site} loader failed:`, error); + } + + const domainEntries = await invoke( + SITES_DOMAINS_LOADER, + { sitename: site }, + apiKey, + ).catch(() => [] as DomainEntry[]); + + const domains = domainEntries.map((d) => extractHostname(d.domain)); + + return { + sitename: site, + hostname: domains[0] ?? "", + domains, + experiments: experiments ?? [], + ...(error ? { error } : {}), + }; + }, +}); + +// ─── 2. Create tool ──────────────────────────────────────────────────────────── + +/** Best-effort user email from the JWT, used for the `createdBy` column. */ +function callerEmail(apiKey: string): string | undefined { + const payload = decodeJwtPayload(apiKey); + const user = payload?.user as Record | undefined; + const email = user?.email ?? payload?.email; + return typeof email === "string" ? email : undefined; +} + +/** + * create_experiment — inserts a new experiment row so it shows up in the list. + * Mirrors the admin `createExperiment` hook (Experiments/hooks.ts), minus the + * traffic-split matcher block, which is configured separately in the editor. + */ +export const createExperimentTool = createTool({ + id: "create_experiment", + title: "Create A/B Test", + description: + "Create a new A/B test experiment for the configured site. Returns the created experiment row.", + inputSchema: z.object({ + name: z.string().min(1).describe("Experiment name"), + description: z.string().optional().describe("Optional description"), + }), + outputSchema: z.object({ + experiment: experimentSchema.nullable(), + error: z.string().optional(), + }), + annotations: { readOnlyHint: false, destructiveHint: false }, + execute: async ({ context }, ctx) => { + const { apiKey, site } = getConfig(ctx); + const { name, description } = context; + + try { + // insertExperiment returns a Supabase PostgrestResponse (`.insert().select()`): + // `{ data: [row], error }` — NOT the rows directly. + const res = await invoke<{ + data?: z.infer[] | null; + error?: { message?: string } | null; + } | null>( + EXPERIMENTS_CREATE_ACTION, + { + site, + name, + description: description ?? "", + status: "draft", + startedAt: new Date().toISOString(), + createdBy: callerEmail(apiKey), + variants: [{ name }], + }, + apiKey, + ); + + if (res?.error) { + console.error(`[create_experiment] site=${site} error:`, res.error); + return { + experiment: null, + error: res.error.message ?? "Insert failed.", + }; + } + + const experiment = res?.data?.[0] ?? null; + if (!experiment) { + return { + experiment: null, + error: + "Experiment was not created (no row returned — likely blocked by permissions).", + }; + } + return { experiment }; + } catch (e) { + const error = e instanceof Error ? e.message : String(e); + console.error(`[create_experiment] site=${site} failed:`, error); + return { experiment: null, error }; + } + }, +}); + +// ─── 3. Results tool — analytics + A/B statistics ───────────────────────────── + +type Filter = [string, string, string[]]; +type DateRange = string | [string, string]; +type AggregateRow = { metrics: number[] }; +type TimeseriesRow = { dimensions: string[]; metrics: number[] }; + +function buildFilters(filters: z.infer): { + tuples: Filter[]; + dimensions: string[]; +} { + const tuples: Filter[] = []; + if (filters?.devices?.length) { + tuples.push(["is", "visit:device", filters.devices]); + } + if (filters?.browsers?.length) { + tuples.push(["is", "visit:browser", filters.browsers]); + } + if (filters?.os?.length) { + tuples.push(["is", "visit:os", filters.os]); + } + return { tuples, dimensions: tuples.map((f) => f[1]) }; +} + +function resolveDateRange(dr: z.infer): DateRange { + return dr.type === "custom" ? [dr.from, dr.to] : dr.value; +} + +/** + * experiment_results — reproduces the analytics + statistics the admin + * Experiments view computes client-side, but server-side so the UI just renders. + * See `deco-sites/admin/.../views/Experiments/Experiments.tsx` for the original. + */ +export const experimentResultsTool = createTool({ + id: "experiment_results", + description: + "Fetch A/B test results for an experiment: per-goal conversions for each variant, a daily timeseries for the selected goal, and the computed statistics (participants, target sample size, and probability each variant is best).", + inputSchema: z.object({ + hostname: z.string().describe("Site hostname (e.g. www.example.com)"), + testName: z + .string() + .describe("Experiment name — the event prop key variants are split on"), + dateRange: dateRangeSchema, + goals: z + .array(z.string()) + .describe("Active goals to aggregate conversions for"), + goalOnDash: z + .string() + .describe("Goal plotted in the timeseries / used for the statistics"), + filters: filtersSchema, + }), + outputSchema: z.object({ + visitors: z.object({ default: z.number(), variant: z.number() }), + goals: z.array( + z.object({ + goal: z.string(), + default: z.number(), + variant: z.number(), + }), + ), + timeseries: z.array( + z.object({ + date: z.string(), + default: z.number(), + variant: z.number(), + }), + ), + stats: z.object({ + totalParticipants: z.number(), + sampleSize: z.number(), + probabilityVariantBest: z.number(), + probabilityDefaultBest: z.number(), + }), + }), + annotations: { readOnlyHint: true, destructiveHint: false }, + execute: async ({ context }, ctx) => { + const { apiKey, site } = getConfig(ctx); + const { hostname, testName, goalOnDash } = context; + const dateRange = resolveDateRange(context.dateRange); + const { tuples: filterTuples, dimensions: filterDims } = buildFilters( + context.filters, + ); + const propKey = `event:props:${testName}`; + + // Always include the visitors baseline — the statistics depend on it. + const goals = Array.from(new Set([VISITORS_GOAL, ...context.goals])); + + // The variant flag lives in the event prop: "true" = test, "false" = default. + const aggregate = (goal: string, variant: string) => + invoke( + ANALYTICS_AGGREGATE_LOADER, + { + sitename: site, + hostname, + query: { + date_range: dateRange, + metrics: ["visitors"], + dimensions: [ + propKey, + ...(goal === VISITORS_GOAL ? [] : ["event:goal"]), + ...filterDims, + ], + filters: [ + ["is", propKey, [variant]], + ...(goal === VISITORS_GOAL ? [] : [["is", "event:goal", [goal]]]), + ...filterTuples, + ], + }, + }, + apiKey, + ) + .then((rows) => rows?.[0]?.metrics?.[0] ?? 0) + .catch(() => 0); + + const timeseries = (variant: string) => + invoke( + ANALYTICS_TIMESERIES_LOADER, + { + sitename: site, + hostname, + query: { + date_range: dateRange, + metrics: ["visitors"], + dimensions: [ + "time:day", + propKey, + ...(goalOnDash === VISITORS_GOAL ? [] : ["event:goal"]), + ...filterDims, + ], + filters: [ + ["is", propKey, [variant]], + ...(goalOnDash === VISITORS_GOAL + ? [] + : [["is", "event:goal", [goalOnDash]]]), + ...filterTuples, + ], + }, + }, + apiKey, + ).catch(() => [] as TimeseriesRow[]); + + // Fire every request concurrently: 2 variants per goal + 2 timeseries. + const [goalCounts, tsDefault, tsVariant] = await Promise.all([ + Promise.all( + goals.map(async (goal) => ({ + goal, + variant: await aggregate(goal, "true"), + default: await aggregate(goal, "false"), + })), + ), + timeseries("false"), + timeseries("true"), + ]); + + const visitorsRow = goalCounts.find((g) => g.goal === VISITORS_GOAL); + const visitors = { + default: visitorsRow?.default ?? 0, + variant: visitorsRow?.variant ?? 0, + }; + + // Merge the two variant timeseries by day so the UI can plot them together. + const byDate = new Map(); + for (const row of tsDefault) { + const date = row.dimensions?.[0] ?? ""; + const entry = byDate.get(date) ?? { default: 0, variant: 0 }; + entry.default = row.metrics?.[0] ?? 0; + byDate.set(date, entry); + } + for (const row of tsVariant) { + const date = row.dimensions?.[0] ?? ""; + const entry = byDate.get(date) ?? { default: 0, variant: 0 }; + entry.variant = row.metrics?.[0] ?? 0; + byDate.set(date, entry); + } + const mergedTimeseries = Array.from(byDate.entries()) + .map(([date, v]) => ({ date, ...v })) + .sort((a, b) => a.date.localeCompare(b.date)); + + // ── statistics (see api/lib/ab-test.ts) ──────────────────────────────── + const successDefault = tsDefault.reduce( + (acc, r) => acc + (r.metrics?.[0] ?? 0), + 0, + ); + const successVariant = tsVariant.reduce( + (acc, r) => acc + (r.metrics?.[0] ?? 0), + 0, + ); + const defaultVariant: Variant = { + successes: successDefault, + total: visitors.default, + }; + const testVariant: Variant = { + successes: successVariant, + total: visitors.variant, + }; + + const rawSampleSize = sampleSizeOf(defaultVariant, testVariant); + const sampleSize = + rawSampleSize == null || Number.isNaN(rawSampleSize) + ? 1000 + : rawSampleSize; + const probabilityVariantBest = pBetter(defaultVariant, testVariant); + + return { + visitors, + goals: goalCounts, + timeseries: mergedTimeseries, + stats: { + totalParticipants: visitors.default + visitors.variant, + sampleSize, + probabilityVariantBest, + probabilityDefaultBest: 1 - probabilityVariantBest, + }, + }; + }, +}); diff --git a/api/tools/index.ts b/api/tools/index.ts index d397f2d..5258271 100644 --- a/api/tools/index.ts +++ b/api/tools/index.ts @@ -1,5 +1,10 @@ import { analyticsQueryTool } from "./analytics-query.ts"; import { assetsTool, deleteAssetTool, uploadAssetTool } from "./assets.ts"; +import { + createExperimentTool, + experimentResultsTool, + listExperimentsTool, +} from "./experiments.ts"; import { getErrorPatternsTool, getErrorRateSeriesTool, @@ -44,4 +49,7 @@ export const tools = [ getErrorPatternsTool, getErrorsOverTimeTool, getErrorRateSeriesTool, + listExperimentsTool, + experimentResultsTool, + createExperimentTool, ]; diff --git a/web/router.tsx b/web/router.tsx index d160bac..134dc12 100644 --- a/web/router.tsx +++ b/web/router.tsx @@ -8,6 +8,7 @@ import { } from "@tanstack/react-router"; import { useMcpHostContext, useMcpState } from "./context.tsx"; import AssetsPage from "./tools/assets/index.tsx"; +import ExperimentsPage from "./tools/experiments/index.tsx"; import LogsPage from "./tools/logs/index.tsx"; import MonitorPage from "./tools/monitor/index.tsx"; import ReleasesPage from "./tools/releases/index.tsx"; @@ -17,6 +18,7 @@ const TOOL_PAGES: Record = { get_logs_data: LogsPage, get_monitor_data: MonitorPage, list_releases: ReleasesPage, + list_experiments: ExperimentsPage, }; function ToolRouter() { diff --git a/web/tools/experiments/index.tsx b/web/tools/experiments/index.tsx new file mode 100644 index 0000000..f8cac17 --- /dev/null +++ b/web/tools/experiments/index.tsx @@ -0,0 +1,1082 @@ +import { ArrowLeft, Filter, FlaskConical, Plus } from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { + Bar, + BarChart, + CartesianGrid, + Legend, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; +import { Button } from "@/components/ui/button.tsx"; +import { Card, CardContent } from "@/components/ui/card.tsx"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog.tsx"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu.tsx"; +import { Input } from "@/components/ui/input.tsx"; +import { Label } from "@/components/ui/label.tsx"; +import { Progress } from "@/components/ui/progress.tsx"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select.tsx"; +import { Skeleton } from "@/components/ui/skeleton.tsx"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table.tsx"; +import { + Tabs, + TabsContent, + TabsList, + TabsTrigger, +} from "@/components/ui/tabs.tsx"; +import { Textarea } from "@/components/ui/textarea.tsx"; +import { useMcpApp, useMcpState } from "@/context.tsx"; + +// ─── types (mirror the experiment_results tool output) ──────────────────────── + +interface Experiment { + id: number; + name: string; + status?: string | null; + startedAt?: string | null; + endedAt?: string | null; + custom_goals?: string[] | null; +} + +interface ExperimentsToolResult { + sitename: string; + hostname: string; + domains: string[]; + experiments: Experiment[]; + error?: string; +} + +interface GoalCount { + goal: string; + default: number; + variant: number; +} + +interface ResultsData { + visitors: { default: number; variant: number }; + goals: GoalCount[]; + timeseries: { date: string; default: number; variant: number }[]; + stats: { + totalParticipants: number; + sampleSize: number; + probabilityVariantBest: number; + probabilityDefaultBest: number; + }; +} + +type Period = "day" | "7d" | "30d" | "month" | "6mo" | "12mo" | "custom"; + +// ─── constants (kept in sync with the admin Experiments view) ───────────────── + +const DEFAULT_COLOR = "#05DAA7"; +const VARIANT_COLOR = "#CA7AD1"; + +const PERIOD_LABELS: Record = { + day: "Day", + "7d": "Last Week", + "30d": "Last 30 Days", + month: "Current Month", + "6mo": "Last 6 Months", + "12mo": "Last Year", + custom: "Custom Date", +}; + +const PERIODS: Period[] = [ + "day", + "7d", + "30d", + "month", + "6mo", + "12mo", + "custom", +]; + +const DEFAULT_GOALS = [ + "visitors", + "view_item_list", + "view_item", + "select_promotion", + "add_to_cart", + "begin_checkout", + "Visit /checkout", + "Visit /checkout#/cart", + "Visit /checkout#/shipping", + "Visit /checkout#/profile", + "Visit /checkout#/email", + "Visit /checkout#/payment", + "Visit /checkout/orderPlaced", +]; + +const DEFAULT_ACTIVE_GOALS = [ + "visitors", + "view_item", + "Visit /checkout/orderPlaced", +]; + +const DEVICES = ["Mobile", "Desktop", "Tablet"]; +const BROWSERS = [ + "Chrome", + "Mobile App", + "Safari", + "Samsung Browser", + "Microsoft Edge", + "Firefox", + "Other", +]; +const OS = ["Android", "iOS", "Windows", "Mac"]; + +// ─── variant tags ───────────────────────────────────────────────────────────── + +function VariantTag({ variant }: { variant: "A" | "B" }) { + return ( + + {variant} + + ); +} + +// ─── multi-select filter dropdown ───────────────────────────────────────────── + +function FilterDropdown({ + label, + options, + selected, + onToggle, +}: { + label: string; + options: string[]; + selected: string[]; + onToggle: (value: string) => void; +}) { + return ( + + + + + + {label} + + {options.map((option) => ( + onToggle(option)} + onSelect={(e) => e.preventDefault()} + > + {option} + + ))} + + + ); +} + +// ─── list view ──────────────────────────────────────────────────────────────── + +function formatDate(value?: string | null): string { + if (!value) return "-"; + const d = new Date(value); + return Number.isNaN(d.getTime()) ? "-" : d.toLocaleDateString(); +} + +function ExperimentsList({ + experiments, + onSelect, + sitename, + error, +}: { + experiments: Experiment[]; + onSelect: (experiment: Experiment) => void; + sitename?: string; + error?: string; +}) { + if (!experiments.length) { + return ( +
+ + {error ? ( + <> +

+ Couldn't load experiments. +

+

{error}

+ + ) : ( +

+ No experiments found for site{" "} + {sitename ?? "?"}. +

+ )} +
+ ); + } + + return ( +
+ + + + Name + Created + Ended + + + + {experiments.map((experiment) => ( + onSelect(experiment)} + > + {experiment.name} + + {formatDate(experiment.startedAt)} + + + {formatDate(experiment.endedAt)} + + + ))} + +
+
+ ); +} + +// ─── results view ───────────────────────────────────────────────────────────── + +interface Goal { + name: string; + active: boolean; +} + +function sortByTotal(goals: GoalCount[]): GoalCount[] { + return [...goals].sort( + (a, b) => b.default + b.variant - (a.default + a.variant), + ); +} + +function percentage(part: number, total: number): number { + if (!part || !total) return 0; + return Number(((part / total) * 100).toFixed(2)); +} + +function ResultsView({ + experiment, + domains, + initialHostname, + callTool, +}: { + experiment: Experiment; + domains: string[]; + initialHostname: string; + callTool: ( + name: string, + args: Record, + ) => Promise; + onBack: () => void; +}) { + const [hostname, setHostname] = useState(initialHostname); + const [period, setPeriod] = useState("30d"); + const [customFrom, setCustomFrom] = useState(""); + const [customTo, setCustomTo] = useState(""); + const [showFilters, setShowFilters] = useState(false); + const [devices, setDevices] = useState([]); + const [browsers, setBrowsers] = useState([]); + const [os, setOs] = useState([]); + + const [goals, setGoals] = useState(() => + [...DEFAULT_GOALS, ...(experiment.custom_goals ?? [])].map((name) => ({ + name, + active: + DEFAULT_ACTIVE_GOALS.includes(name) || + (experiment.custom_goals ?? []).includes(name) || + name === "visitors", + })), + ); + const [goalOnDash, setGoalOnDash] = useState("begin_checkout"); + + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + + const activeGoals = useMemo( + () => goals.filter((g) => g.active).map((g) => g.name), + [goals], + ); + // Serialize for stable effect deps without re-fetching on identity changes. + const activeGoalsKey = activeGoals.join(","); + const filtersKey = `${devices.join(",")}|${browsers.join(",")}|${os.join(",")}`; + + useEffect(() => { + if (!hostname) return; + if (period === "custom" && (!customFrom || !customTo)) return; + + let cancelled = false; + setLoading(true); + + const dateRange = + period === "custom" + ? { type: "custom" as const, from: customFrom, to: customTo } + : { type: "preset" as const, value: period }; + + callTool("experiment_results", { + hostname, + testName: experiment.name, + dateRange, + goals: activeGoals, + goalOnDash, + filters: { devices, browsers, os }, + }).then((result) => { + if (cancelled) return; + setData(result); + setLoading(false); + }); + + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + hostname, + period, + customFrom, + customTo, + activeGoalsKey, + filtersKey, + goalOnDash, + experiment.name, + callTool, + ]); + + const sortedGoals = useMemo( + () => + data ? sortByTotal(data.goals.filter((g) => g.goal !== "visitors")) : [], + [data], + ); + + const stats = data?.stats; + const visitors = data?.visitors ?? { default: 0, variant: 0 }; + const sampleSize = stats?.sampleSize ?? 1000; + const progress = stats + ? Math.min(100, (stats.totalParticipants / (sampleSize || 1000)) * 100) + : 0; + + const toggle = useCallback( + (setter: React.Dispatch>, value: string) => + setter((prev) => + prev.includes(value) + ? prev.filter((v) => v !== value) + : [...prev, value], + ), + [], + ); + + const timeseriesData = + data?.timeseries.map((point) => ({ + date: point.date, + Default: point.default, + "Test Variant 1": point.variant, + })) ?? []; + + const barData = sortedGoals.map((g) => ({ + goal: g.goal, + Default: percentage(g.default, visitors.default), + "Test Variant 1": percentage(g.variant, visitors.variant), + })); + + return ( +
+ {/* header */} +
+
+ {experiment.name} +
+
+ {domains.length > 0 && ( + + )} + + +
+
+ + {period === "custom" && ( +
+ setCustomFrom(e.target.value)} + className="border border-border rounded-md px-2 py-1 text-sm bg-background" + /> + → + setCustomTo(e.target.value)} + className="border border-border rounded-md px-2 py-1 text-sm bg-background" + /> +
+ )} + + {showFilters && ( +
+ toggle(setDevices, v)} + /> + toggle(setBrowsers, v)} + /> + toggle(setOs, v)} + /> +
+ )} + + {/* progress card */} + + + +
+

Experiment in progress

+

+ It's too early to tell which variant is better as the results are + not statistically significant and may still change. +

+
+
+
+ + {loading ? "…" : (stats?.totalParticipants ?? 0)} participants + have seen + + + Goal: {loading ? "…" : sampleSize.toFixed()} participants + +
+ +
+ + + + {/* variant probability + timeseries */} +
+
+

Variant results

+ +
+ + +
+ + +
+
+ {loading ? ( + + ) : ( + + + + + + + + + + + + )} +
+ + +
+ + {/* goals bar chart */} +
+
+

Goals

+ +
+ + + {loading ? ( + + ) : ( + + + + + `${v}%`} + /> + `${value}%`} + contentStyle={{ + backgroundColor: "hsl(var(--background))", + border: "1px solid hsl(var(--border))", + borderRadius: "8px", + fontSize: 12, + }} + /> + + + + + + )} + + +
+ + {/* tables */} + + + Funnel + By Goal + + +
+ + + + Goal + + + Default + + + + + Test Variant 1 + + + + + + {sortedGoals.map((g) => ( + + {g.goal} + + + + + + + + ))} + +
+
+ + + {sortedGoals.map((g) => ( +
+

{g.goal}

+
+ + + + Variant + Visitors + {g.goal} + + + + + + + Default + + + {visitors.default} + {g.default} + + + + + Test Variant 1 + + + {visitors.variant} + {g.variant} + + +
+
+
+ ))} + + +
+ ); +} + +function ProbabilityBlock({ + variant, + label, + probability, + color, + loading, +}: { + variant: "A" | "B"; + label: string; + probability: number; + color: string; + loading: boolean; +}) { + return ( +
+ + + {label} + + + + Probability that this variant is the best:{" "} + {loading ? "…" : `${(probability * 100).toFixed(1)}%`} + +
+ ); +} + +function ConversionCell({ count, total }: { count: number; total: number }) { + if (!count) return 0; + return ( + + {count} + {total > 0 && ( + + {((count / total) * 100).toFixed(2)}% + + )} + + ); +} + +function GoalsDropdown({ + goals, + setGoals, +}: { + goals: Goal[]; + setGoals: React.Dispatch>; +}) { + const activeCount = goals.filter((g) => g.active).length; + return ( + + + + + + {goals.map((goal) => ( + + setGoals((prev) => + prev.map((g) => + g.name === goal.name ? { ...g, active: !g.active } : g, + ), + ) + } + onSelect={(e) => e.preventDefault()} + > + {goal.name} + + ))} + + + ); +} + +// ─── new experiment dialog ──────────────────────────────────────────────────── + +interface CreateResult { + experiment: Experiment | null; + error?: string; +} + +function NewExperimentDialog({ + open, + onOpenChange, + callTool, + onCreated, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + callTool: ( + name: string, + args: Record, + ) => Promise; + onCreated: () => void | Promise; +}) { + const [name, setName] = useState(""); + const [description, setDescription] = useState(""); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + + // Reset the form whenever the dialog opens. + useEffect(() => { + if (open) { + setName(""); + setDescription(""); + setError(null); + setSubmitting(false); + } + }, [open]); + + const submit = async () => { + if (!name.trim() || submitting) return; + setSubmitting(true); + setError(null); + const res = await callTool("create_experiment", { + name: name.trim(), + description: description.trim() || undefined, + }); + setSubmitting(false); + if (!res || res.error || !res.experiment) { + setError(res?.error ?? "Failed to create experiment."); + return; + } + onOpenChange(false); + await onCreated(); + }; + + return ( + + + + Create new experiment + + Give your A/B test a name. You can configure the traffic split in + the editor afterwards. + + +
+
+ + setName(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") submit(); + }} + /> +
+
+ +