diff --git a/backend/app/models/PageView.py b/backend/app/models/PageView.py index b0d9a6f..d3db50e 100644 --- a/backend/app/models/PageView.py +++ b/backend/app/models/PageView.py @@ -26,6 +26,10 @@ class PageView(Base): __table_args__ = ( Index("ix_page_view_created_at", "created_at"), Index("ix_page_view_path", "path"), + # Matches the navigation-flow query's ORDER BY (see + # app/routers/admin/analytics.py), which reconstructs sessions per + # visitor and otherwise forces a full sort of the date-range scan. + Index("ix_page_view_visitor_created", "visitor_hash", "created_at", "id"), ) def __repr__(self) -> str: diff --git a/backend/script/init_db.py b/backend/script/init_db.py index 771a230..fed6778 100644 --- a/backend/script/init_db.py +++ b/backend/script/init_db.py @@ -11,11 +11,12 @@ from __future__ import annotations import os +import re import sys from sqlalchemy import inspect, or_, text from sqlalchemy.exc import IntegrityError -from sqlalchemy.schema import CreateColumn +from sqlalchemy.schema import CreateColumn, CreateIndex from app.database import ( Base, @@ -67,6 +68,33 @@ def sync_missing_columns() -> None: print(f"Added missing column {table.name}.{column.name}") +def sync_missing_indexes() -> None: + """Add indexes present on the ORM models but missing from deployed tables. + + create_all() only creates indexes when it creates the table itself; it + never alters a table that's already there (same gap sync_missing_columns() + covers for columns). Runs CREATE INDEX CONCURRENTLY so building the index + on a large existing table doesn't hold a lock against writes. + """ + inspector = inspect(engine) + existing_tables = set(inspector.get_table_names()) + + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + for table in Base.metadata.sorted_tables: + if table.name not in existing_tables: + continue # just created by create_missing_tables() + + existing_indexes = {ix["name"] for ix in inspector.get_indexes(table.name)} + for index in table.indexes: + if index.name in existing_indexes: + continue + + index_ddl = str(CreateIndex(index).compile(dialect=engine.dialect)) + index_ddl = re.sub(r"^CREATE( UNIQUE)? INDEX", r"CREATE\1 INDEX CONCURRENTLY", index_ddl) + conn.execute(text(index_ddl)) + print(f"Added missing index {index.name} on {table.name}") + + def bootstrap_initial_admin() -> None: email = os.getenv("INITIAL_ADMIN_EMAIL") onyen = os.getenv("INITIAL_ADMIN_ONYEN") @@ -132,5 +160,6 @@ def bootstrap_initial_admin() -> None: print("Initializing deployed database") create_missing_tables() sync_missing_columns() + sync_missing_indexes() bootstrap_initial_admin() print("Database initialization complete") diff --git a/frontend/src/app/admin/analytics/page.tsx b/frontend/src/app/admin/analytics/page.tsx index ad6c2ee..26a83d0 100644 --- a/frontend/src/app/admin/analytics/page.tsx +++ b/frontend/src/app/admin/analytics/page.tsx @@ -24,35 +24,58 @@ const ACTIVE_USERS_POLL_MS = 30_000; export default function AdminAnalyticsPage() { const [days, setDays] = useState(7); const [summary, setSummary] = useState(null); - const [navigationFlow, setNavigationFlow] = useState(null); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); const [activeUsers, setActiveUsers] = useState(null); + const [navigationFlow, setNavigationFlow] = useState(null); + const [isFlowLoading, setIsFlowLoading] = useState(true); + + // Kept separate from the navigation-flow fetch below: the flow query scans + // and sorts every pageview in range, so it can lag well behind the summary + // stats. Gating the whole page on both left visitors staring at a blank + // spinner for the slower of the two; the summary and line chart should + // render as soon as they're ready. useEffect(() => { let isMounted = true; - async function fetchData() { + async function fetchSummary() { setIsLoading(true); setError(null); try { - const [summaryData, flowData] = await Promise.all([ - getAnalyticsSummary(days), - getNavigationFlow(days), - ]); - if (isMounted) { - setSummary(summaryData); - setNavigationFlow(flowData); - } + const summaryData = await getAnalyticsSummary(days); + if (isMounted) setSummary(summaryData); } catch (err) { - console.error("Failed to fetch analytics data:", err); + console.error("Failed to fetch analytics summary:", err); if (isMounted) setError("Failed to load analytics data."); } finally { if (isMounted) setIsLoading(false); } } - fetchData(); + fetchSummary(); + + return () => { + isMounted = false; + }; + }, [days]); + + useEffect(() => { + let isMounted = true; + + async function fetchNavigationFlow() { + setIsFlowLoading(true); + try { + const flowData = await getNavigationFlow(days); + if (isMounted) setNavigationFlow(flowData); + } catch (err) { + console.error("Failed to fetch navigation flow:", err); + } finally { + if (isMounted) setIsFlowLoading(false); + } + } + + fetchNavigationFlow(); return () => { isMounted = false; @@ -109,7 +132,12 @@ export default function AdminAnalyticsPage() { ) : error ? (
{error}
) : summary ? ( - + ) : null} diff --git a/frontend/src/components/admin/AnalyticsCharts.tsx b/frontend/src/components/admin/AnalyticsCharts.tsx index f0d0817..63b5cfe 100644 --- a/frontend/src/components/admin/AnalyticsCharts.tsx +++ b/frontend/src/components/admin/AnalyticsCharts.tsx @@ -1,18 +1,20 @@ "use client"; -import type { AnalyticsSummary, DailyPageViewCount, NavigationFlow, NavigationFlowLink } from "@/types/admin"; -import { - CartesianGrid, - Layer, - Line, - LineChart, - Rectangle, - ResponsiveContainer, - Sankey, - Tooltip, - XAxis, - YAxis, -} from "recharts"; +import type { AnalyticsSummary, DailyPageViewCount, NavigationFlow } from "@/types/admin"; +import dynamic from "next/dynamic"; +import { CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; + +// Code-split from the initial analytics bundle: the flow diagram fetches and +// computes independently of the summary stats above it (see +// app/admin/analytics/page.tsx), so it shouldn't block their first paint. +const NavigationFlowChart = dynamic(() => import("./NavigationFlowChart"), { + ssr: false, + loading: () => ( +
+ Loading flow… +
+ ), +}); function StatTile({ label, value }: { label: string; value: number | null }) { return ( @@ -113,111 +115,16 @@ function AnalyticsTooltip({ ); } -const SANKEY_WIDTH = 760; -const SANKEY_HEIGHT = 420; -const SESSION_START_SENTINEL = "__start__"; -const SESSION_START_LABEL = "Session Start"; - -function truncateLabel(name: string, max = 28): string { - return name.length > max ? `${name.slice(0, max - 1)}…` : name; -} - -function buildSankeyData(links: NavigationFlowLink[]) { - const nodeIndex = new Map(); - const nodes: { name: string }[] = []; - - function indexFor(rawName: string): number { - let index = nodeIndex.get(rawName); - if (index === undefined) { - index = nodes.length; - nodeIndex.set(rawName, index); - nodes.push({ name: rawName === SESSION_START_SENTINEL ? SESSION_START_LABEL : rawName }); - } - return index; - } - - return { - nodes, - links: links.map((link) => ({ - source: indexFor(link.source), - target: indexFor(link.target), - value: link.count, - })), - }; -} - -function renderFlowNode({ - x, - y, - width, - height, - payload, -}: { - x: number; - y: number; - width: number; - height: number; - payload: { name: string }; -}) { - const isRightHalf = x + width / 2 > SANKEY_WIDTH / 2; - return ( - - - - {truncateLabel(payload.name)} - - - ); -} - -function NavigationFlowChart({ data }: { data: NavigationFlow | null }) { - return ( -
-

- Navigation Flow - {data && data.total_sessions > 0 && ( - - ({data.total_sessions.toLocaleString()} sessions analyzed) - - )} -

- {!data || data.links.length === 0 ? ( -

No data yet.

- ) : ( -
- - - -
- )} -
- ); -} - export function AnalyticsCharts({ summary, activeUsers, navigationFlow, + isNavigationFlowLoading, }: { summary: AnalyticsSummary; activeUsers: number | null; navigationFlow: NavigationFlow | null; + isNavigationFlowLoading?: boolean; }) { const hourly = summary.range_days <= 1; @@ -269,7 +176,7 @@ export function AnalyticsCharts({ /> - + ); } diff --git a/frontend/src/components/admin/NavigationFlowChart.tsx b/frontend/src/components/admin/NavigationFlowChart.tsx new file mode 100644 index 0000000..afcabdc --- /dev/null +++ b/frontend/src/components/admin/NavigationFlowChart.tsx @@ -0,0 +1,266 @@ +"use client"; + +import type { NavigationFlow, NavigationFlowLink } from "@/types/admin"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { Layer, Rectangle, Sankey, Tooltip } from "recharts"; +import type { SankeyLinkProps, SankeyNodeProps } from "recharts"; + +const SANKEY_HEIGHT = 440; +const MIN_SANKEY_WIDTH = 600; +const SESSION_START_SENTINEL = "__start__"; +const SESSION_START_LABEL = "Session Start"; + +// First three slots of the validated categorical palette (dataviz skill's +// documented default): blue / orange / aqua clear the all-pairs CVD floor, +// which matters here because flow bands from different roots can sit +// directly beside each other (unlike bars/lines, adjacency isn't fixed). +// A 4th+ root folds into OTHER_COLOR rather than cycling back through them. +const FLOW_PALETTE = ["#2a78d6", "#eb6834", "#1baf7a"]; +const OTHER_COLOR = "#94a3b8"; +const NODE_TEXT_COLOR = "#334155"; + +function truncateLabel(name: string, max = 32): string { + return name.length > max ? `${name.slice(0, max - 1)}…` : name; +} + +interface FlowNode { + name: string; + color: string; + value: number; +} + +interface FlowLink { + source: number; + target: number; + value: number; +} + +// Colors every node by the root (session-start or direct-entry page) it +// descends from, so a viewer can trace one flow's path across the diagram +// instead of every band reading as the same flat gray. +function assignNodeColors(nodeCount: number, links: FlowLink[]): string[] { + const adjacency = new Map(); + const hasIncoming = new Set(); + for (const link of links) { + hasIncoming.add(link.target); + const existing = adjacency.get(link.source); + if (existing) existing.push(link.target); + else adjacency.set(link.source, [link.target]); + } + + const roots = Array.from({ length: nodeCount }, (_, i) => i).filter((i) => !hasIncoming.has(i)); + const colors = new Array(nodeCount).fill(OTHER_COLOR); + const colored = new Array(nodeCount).fill(false); + + roots.forEach((root, rootIndex) => { + const color = rootIndex < FLOW_PALETTE.length ? FLOW_PALETTE[rootIndex] : OTHER_COLOR; + const queue = [root]; + while (queue.length > 0) { + const node = queue.shift(); + if (node === undefined || colored[node]) continue; + colored[node] = true; + colors[node] = color; + for (const next of adjacency.get(node) ?? []) { + if (!colored[next]) queue.push(next); + } + } + }); + + return colors; +} + +function buildSankeyData(rawLinks: NavigationFlowLink[]) { + const nodeIndex = new Map(); + const names: string[] = []; + + function indexFor(rawName: string): number { + let index = nodeIndex.get(rawName); + if (index === undefined) { + index = names.length; + nodeIndex.set(rawName, index); + names.push(rawName === SESSION_START_SENTINEL ? SESSION_START_LABEL : rawName); + } + return index; + } + + const links: FlowLink[] = rawLinks.map((link) => ({ + source: indexFor(link.source), + target: indexFor(link.target), + value: link.count, + })); + + const colors = assignNodeColors(names.length, links); + + // Each node's own throughput: prefer incoming (how many sessions reached + // this page), falling back to outgoing for root nodes with no incoming edge. + const incoming = new Array(names.length).fill(0); + const outgoing = new Array(names.length).fill(0); + for (const link of links) { + outgoing[link.source] += link.value; + incoming[link.target] += link.value; + } + + const nodes: FlowNode[] = names.map((name, i) => ({ + name, + color: colors[i], + value: incoming[i] > 0 ? incoming[i] : outgoing[i], + })); + + return { nodes, links }; +} + +function renderFlowNode(containerWidth: number) { + return function FlowNodeShape({ x, y, width, height, payload: rawPayload }: SankeyNodeProps) { + // recharts' SankeyNode type doesn't know about the `color`/`value` fields + // we attach in buildSankeyData, but it passes our original node objects + // straight through as payload, so this cast reflects the real runtime shape. + const payload = rawPayload as unknown as FlowNode; + const isRightHalf = x + width / 2 > containerWidth / 2; + const label = `${truncateLabel(payload.name)} · ${payload.value.toLocaleString()}`; + // No canvas measurement available here, so estimate the label chip's + // width from character count — just needs to comfortably cover the text. + const estTextWidth = label.length * 6.4 + 10; + const textX = isRightHalf ? x - 6 : x + width + 6; + const chipX = isRightHalf ? textX - estTextWidth : textX; + + return ( + + + + + {label} + + + ); + }; +} + +function renderFlowLink({ + sourceX, + sourceY, + sourceControlX, + targetX, + targetY, + targetControlX, + linkWidth, + payload: rawPayload, +}: SankeyLinkProps) { + // Same cast as renderFlowNode: payload.source/target are our own node + // objects at runtime, carrying the color field SankeyNode doesn't declare. + const payload = rawPayload as unknown as { source: FlowNode; target: FlowNode }; + return ( + + ); +} + +function FlowTooltip({ + active, + payload, +}: { + active?: boolean; + payload?: { payload: FlowNode | (FlowLink & { source: FlowNode; target: FlowNode }) }[]; +}) { + if (!active || !payload || payload.length === 0) return null; + + const raw = payload[0].payload; + const isLink = "source" in raw && "target" in raw; + const color = isLink ? raw.source.color : raw.color; + const label = isLink ? `${raw.source.name} → ${raw.target.name}` : raw.name; + + return ( +
+
+ + {label} +
+

+ {raw.value.toLocaleString()} session{raw.value === 1 ? "" : "s"} +

+
+ ); +} + +function useMeasuredWidth(fallback: number) { + const ref = useRef(null); + const [width, setWidth] = useState(fallback); + + useEffect(() => { + const el = ref.current; + if (!el) return; + setWidth(el.clientWidth || fallback); + const observer = new ResizeObserver((entries) => { + const entry = entries[0]; + if (entry) setWidth(entry.contentRect.width); + }); + observer.observe(el); + return () => observer.disconnect(); + }, [fallback]); + + return [ref, width] as const; +} + +export default function NavigationFlowChart({ + data, + isLoading, +}: { + data: NavigationFlow | null; + isLoading?: boolean; +}) { + const [containerRef, measuredWidth] = useMeasuredWidth(MIN_SANKEY_WIDTH); + const width = Math.max(measuredWidth, MIN_SANKEY_WIDTH); + const sankeyData = useMemo(() => (data ? buildSankeyData(data.links) : null), [data]); + const nodeRenderer = useMemo(() => renderFlowNode(width), [width]); + + return ( +
+

+ Navigation Flow + {data && data.total_sessions > 0 && ( + + ({data.total_sessions.toLocaleString()} sessions analyzed) + + )} +

+
+ {isLoading ? ( +
Loading flow…
+ ) : !sankeyData || sankeyData.links.length === 0 ? ( +

No data yet.

+ ) : ( + + } /> + + )} +
+
+ ); +}