Add A/B test results tool with interactive UI

api/app.ts

@@ -1,5 +1,6 @@
 import { withRuntime } from "@decocms/runtime";
 import { createAssetsAppResource } from "./resources/assets.ts";
+import { createExperimentsAppResource } from "./resources/experiments.ts";
 import { createLogsAppResource } from "./resources/logs.ts";
 import { createMonitorAppResource } from "./resources/monitor.ts";
 import { createReleasesAppResource } from "./resources/releases.ts";
@@ -100,6 +101,7 @@ export function createApp(opts: CreateAppOptions): Fetcher {
 		tools,
 		resources: [
 			createAssetsAppResource(getClientHTML),
+			createExperimentsAppResource(getClientHTML),
 			createLogsAppResource(getClientHTML),
 			createMonitorAppResource(getClientHTML),
 			createReleasesAppResource(getClientHTML),

api/lib/ab-test.ts

@@ -0,0 +1,100 @@
+/**
+ * A/B test statistics — server-side port of the deco.cx admin util.
+ *
+ * Reference: `deco-sites/admin/utils/statistics/abTest.ts`
+ * (components/spaces/siteEditor/extensions/CMS/views/Experiments/Experiments.tsx
+ * is the sole consumer over there, computing these numbers in the browser).
+ *
+ * The admin version depends on `jstat` purely for `jStat.normal.cdf`. Pulling
+ * the whole ~45KB library in just for one function is wasteful — especially
+ * here, where the client bundle is inlined into a single HTML resource. So we
+ * reimplement the standard-normal CDF with an Abramowitz & Stegun erf
+ * approximation (formula 7.1.26, |error| < 1.5e-7) and keep the rest of the math
+ * identical to the admin implementation. Everything runs inside the MCP tool so
+ * the UI only renders the results.
+ */
+
+export interface Variant {
+	/** Number of successes (e.g. conversions for the selected goal). */
+	successes: number;
+	/** Total participants (e.g. visitors that saw this variant). */
+	total: number;
+}
+
+// alpha = 0.05, one-sided
+const Z_ALPHA = 1.644853;
+// beta = 0.2, one-sided
+const Z_BETA = 0.8416;
+const MIN_SAMPLE_SIZE = 1000;
+
+/**
+ * Standard-normal cumulative distribution function.
+ * Drop-in replacement for `jStat.normal.cdf(x, mean, std)`.
+ */
+export function normalCdf(x: number, mean = 0, std = 1): number {
+	const z = (x - mean) / (std * Math.SQRT2);
+	// erf approximation — Abramowitz & Stegun 7.1.26
+	const t = 1 / (1 + 0.3275911 * Math.abs(z));
+	const y =
+		1 -
+		((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) *
+			t +
+			0.254829592) *
+			t *
+			Math.exp(-z * z);
+	const erf = z >= 0 ? y : -y;
+	return 0.5 * (1 + erf);
+}
+
+function proportion(a: Variant): number {
+	return a.successes / a.total;
+}
+
+function standardError(a: Variant): number {
+	const p = proportion(a);
+	return Math.sqrt((p * (1 - p)) / a.total);
+}
+
+/**
+ * Probability that a random sample from B is greater than one from A —
+ * i.e. the chance the test variant beats the default.
+ */
+export function pBetter(a: Variant, b: Variant): number {
+	const z =
+		-(proportion(b) - proportion(a)) /
+		Math.sqrt(standardError(a) ** 2 + standardError(b) ** 2);
+	const result = 1 - normalCdf(z, 0, 1);
+	return Number.isFinite(result) ? result : 0;
+}
+
+/**
+ * Required sample size to reach significance.
+ * Based on Wiley Series in Probability and Statistics, Chapter 4.
+ * Returns `null` when there isn't enough data yet to establish a control.
+ */
+export function sampleSize(a: Variant, b: Variant, mde = 0.03): number | null {
+	// not enough data to establish a control for the sample size calculation
+	if (a.total < MIN_SAMPLE_SIZE || a.successes === 0) {
+		return null;
+	}
+
+	const r = b.total / a.total;
+
+	const p1 = proportion(a);
+	// use MDE if variant B's proportion would cause a sample size that is too large
+	const p2 = Math.max(p1 * (1 + mde), proportion(b));
+	const p = (p1 + r * p2) / (r + 1);
+
+	const numerator =
+		Z_ALPHA * Math.sqrt((r + 1) * p * (1 - p)) +
+		Z_BETA * Math.sqrt(r * p1 * (1 - p1) + p2 * (1 - p2));
+	const denominator = p2 - p1;
+	const sampleSizeRaw = (numerator / denominator) ** 2 / r;
+
+	const mul =
+		1 + Math.sqrt(1 + (2 * (r + 1)) / (sampleSizeRaw * r * Math.abs(p1 - p2)));
+
+	const size = (sampleSizeRaw * mul ** 2) / 4;
+
+	return Math.ceil(size) * (1 + r);
+}

api/resources/experiments.ts

@@ -0,0 +1,23 @@
+import { createPublicResource } from "@decocms/runtime/tools";
+import { EXPERIMENTS_RESOURCE_URI } from "../tools/experiments.ts";
+
+const RESOURCE_MIME_TYPE = "text/html;profile=mcp-app";
+
+export const createExperimentsAppResource = (
+	getClientHTML: () => Promise<string>,
+) =>
+	createPublicResource({
+		uri: EXPERIMENTS_RESOURCE_URI,
+		name: "A/B Test Results UI",
+		description:
+			"Interactive A/B test results dashboard: variant conversions, timeseries, and significance statistics for deco.cx experiments",
+		mimeType: RESOURCE_MIME_TYPE,
+		read: async () => {
+			const html = await getClientHTML();
+			return {
+				uri: EXPERIMENTS_RESOURCE_URI,
+				mimeType: RESOURCE_MIME_TYPE,
+				text: html,
+			};
+		},
+	});