diff --git a/CLAUDE.md b/CLAUDE.md index ab00aa057..9c4f1983c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -341,6 +341,14 @@ nullable or has a DEFAULT. `ALTER TYPE … ADD VALUE` needs a `.sql.conf` sideca once as SQL_REVIEW_BLOCKED when it changed the outcome) PENDING_AI → REJECTED (routing-policy AUTO_REJECT — AF-379; no review_decisions row, audited via QueryAutoRejectedEvent) + PENDING_AI → REJECTED (bytes-scanned cap — #941; the warehouse's pre-flight bytes estimate + exceeds the datasource/grant cap, or there is none and the datasource's + bytes_cap_missing_estimate=REJECT. Decided before routing and the + AI-failed path, no routing_decision row, QueryAutoRejectedEvent with a + null policy id; audited as QUERY_BYTES_SCANNED_CAP_ENFORCED) + PENDING_AI → PENDING_REVIEW (bytes-scanned cap, no estimate, bytes_cap_missing_estimate= + REQUIRE_REVIEW — #941; suppresses the same auto-approve paths as a SQL + review BLOCK, never softens AUTO_REJECT) PENDING_REVIEW → APPROVED or REJECTED (external ticket resolution — AF-453; a channel with bidirectional_sync=true maps a ServiceNow/Jira ticket resolution onto a decision via workflow.api.ExternalDecisionService. System-attributed: @@ -363,7 +371,9 @@ nullable or has a DEFAULT. `ALTER TYPE … ADD VALUE` needs a `.sql.conf` sideca recurrence_next_run_at, records recurrence_halted_reason, and audits RECURRING_SERIES_HALTED) APPROVED → EXECUTED (break-glass run — audit action QUERY_BREAK_GLASS_EXECUTED — AF-385) - APPROVED → FAILED (execution error) + APPROVED → FAILED (execution error; also the bytes-scanned cap re-checked just before + execution refusing the run — #941, scheduled / recurring / + break-glass included) ``` Illegal transitions must throw a domain exception, not silently succeed. **Break-glass / diff --git a/README.md b/README.md index 504fed2c4..be8f70ec9 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ A glance at the day-to-day flows engineers and approvers actually use. - **Proxy-first execution** — no user ever holds production credentials; the proxy holds them encrypted and opens connections only after approval. Single SQL statements run with autocommit; multi-statement INSERT/UPDATE/DELETE batches wrapped in `BEGIN; … COMMIT;` execute atomically inside one JDBC transaction (mixed SELECT/DML batches are rejected at parse time), with homogeneous INSERT runs collapsed into JDBC `executeBatch()` for bulk-load throughput. Optional **multi-replica read load balancing**: attach any number of replica endpoints to a datasource and SELECT traffic round-robins across the healthy ones — per-node health checks with circuit-breaker failover skip downed replicas, and only full replica-set exhaustion falls back to the primary (with an audit row). Optional **SELECT result caching**: opt a datasource into a Redis-backed result cache (per-datasource TTL) keyed over the security-rewritten query — masking and row-level security still apply — and invalidated on any proxied write to a referenced table. - **Configurable review workflows** — per-datasource review plans, multi-stage sequential approval chains, optional auto-approve for reads, approval timeouts with auto-reject. Reviewers can be scoped **per-datasource** (directly or via groups) so different teams see only the queues that belong to them. Reviewers going away can set an **out-of-office delegation** naming a colleague to cover their review duty for a window — across queries, governed API calls, and grouped requests — with every decision recording both identities. A delegate can never act on the delegator's own requests, delegation never grants a permission they lack, and it does not chain. Plans can also **escalate** a request that nobody has decided on to the reviewers at its current stage plus your admins before the approval timeout auto-rejects it, and **nudge** those same reviewers on a cadence — both optional, both notify-only, so waiting never changes who may approve. -- **Policy-as-code routing** — ordered, attribute-based routing policies decide a query's path after AI analysis and before reviewers see it: **auto-approve**, **auto-reject**, **require N approvals**, or **escalate**. Conditions match on query type, referenced tables (glob), AI risk level / score, requester role or group, time-of-day / day-of-week, WHERE / LIMIT presence, **query shape** (joins, set operations, subqueries, CTEs, GROUP BY / HAVING, aggregates, window functions), the transactional flag, and the submission **client context** — source IP / CIDR, user-agent, time-since-last-approval, and CI/CD origin (API key or `X-AccessFlow-CI` header) — combined with AND / OR / NOT. Client-context conditions **fail closed** (missing context never auto-approves), so an off-network or stale-approval query escalates to stricter review instead. First match by priority wins; on no match the query falls through to the datasource's review plan. Every automated decision is recorded in the audit log. +- **Policy-as-code routing** — ordered, attribute-based routing policies decide a query's path after AI analysis and before reviewers see it: **auto-approve**, **auto-reject**, **require N approvals**, or **escalate**. Conditions match on query type, referenced tables (glob), AI risk level / score, requester role or group, time-of-day / day-of-week, WHERE / LIMIT presence, **query shape** (joins, set operations, subqueries, CTEs, GROUP BY / HAVING, aggregates, window functions), the transactional flag, the pre-flight estimate (**estimated rows, estimated bytes scanned**, scan type), and the submission **client context** — source IP / CIDR, user-agent, time-since-last-approval, and CI/CD origin (API key or `X-AccessFlow-CI` header) — combined with AND / OR / NOT. Client-context conditions **fail closed** (missing context never auto-approves), so an off-network or stale-approval query escalates to stricter review instead. First match by priority wins; on no match the query falls through to the datasource's review plan. Every automated decision is recorded in the audit log. - **Policy simulator** — dry-run a draft routing, row-security, or masking policy against your own historical query traffic before you save it. AccessFlow replays the window **twice** — once against your current policies, once with the draft applied — and reports the difference between those two runs: how many past queries would change, which users would be affected, and which queries a row predicate would newly filter, deny, or reject outright. It is strictly read-only — no connection to your database, nothing executed, nothing persisted — and it names its own approximations (memberships are read as they are now; an engine that cannot classify a query shape offline is reported as *unclassifiable*, never as safe) instead of implying a precision the data does not have. - **Deterministic SQL review rules (#860)** — a named rule catalog that judges every SQL query alongside the AI verdict and routing policies, and shows its findings **in the editor as you type**. Fourteen built-in rules derived from the parsed statement alone (missing `WHERE` on `UPDATE` / `DELETE`, an always-true `WHERE`, `SELECT *`, unbounded reads, cross joins, leading-wildcard `LIKE`, `DROP` / `TRUNCATE` / DDL, banned functions, protected tables by glob, DML outside a transaction), each at an admin-set **`OFF` / `WARN` / `BLOCK`** severity configured per **environment** (`DEVELOPMENT` / `TEST` / `STAGING` / `PRODUCTION`, plus an organisation default). **`BLOCK` escalates, it never rejects**: a blocking finding suppresses every auto-approve path and sends the query to a human reviewer; a routing auto-reject still rejects. Evaluated synchronously at submission so findings exist even when AI analysis is off or fails, rendered in each reader's language on the query detail, review queue, break-glass retro-review and request-group detail; break-glass runs record findings but are never gated by them, and non-relational engines report *not applicable*. See [`docs/19-sql-review.md`](https://github.com/bablsoft/accessflow/blob/main/docs/19-sql-review.md). - **Just-in-time (JIT) access requests** — users self-request temporary, scoped access to a datasource (read/write/DDL, optional schema/table scope) or an API connection (read/write, optional operation allow-list) for an ISO-8601 duration. Requests flow through the same approval engine, a time-boxed permission is granted on approval, and a clustered scheduler auto-revokes it on expiry (admins can also revoke early). A grant can opt into **query pre-approval**: while it is active, queries it covers skip human review and are auto-approved with the grant recorded as the approval provenance — routing policies, high-risk AI verdicts, and behavioural anomalies still override. @@ -84,6 +84,7 @@ A glance at the day-to-day flows engineers and approvers actually use. - **Query-optimization suggestions** — alongside the risk verdict, the analyzer returns concrete, dialect-aware optimization suggestions: index recommendations (`CREATE INDEX …`) and query rewrites. Each suggestion has a one-click **"Apply as draft"** that pre-fills the editor with the suggested statement and routes it through the normal review pipeline — nothing auto-executes, and the draft is audited as AI-suggested. - **Query playground / dry-run sandbox** — preview a query's impact before submitting it for review. A **Dry run** action returns the engine's execution plan (node type, estimated rows, cost, filters) and a best-effort estimated row impact **without executing or mutating data**, shown in the editor next to the AI panel. It runs through the same governance as a real execution (datasource access, table allow-list, row-level security) but creates no review. Dialect-aware across every engine with a plan concept — relational `EXPLAIN` (PostgreSQL / MySQL / MariaDB / Oracle / SQL Server), MongoDB `explain`, Couchbase / Neo4j `EXPLAIN`, Elasticsearch / OpenSearch query validation; engines without one degrade gracefully. Cloud warehouses additionally report the estimated bytes the query would scan (BigQuery's native dry-run job) — the direct cost signal for bytes-billed engines. SELECT dry-runs prefer the read replica when configured. - **Pre-flight cost & blast-radius estimation (AF-624)** — every submitted query automatically gets a persisted cost estimate before review: the engine's own `EXPLAIN` plan (estimated rows, scan type, cost) plus, for UPDATE/DELETE, an **exact affected-row count** computed with a governed, non-mutating count (relational `SELECT COUNT(*)` rewrite; MongoDB `countDocuments`, SQL++ `COUNT(*)`, Cypher `count(*)`, Elasticsearch `_count`). Reviewers see it on the query detail page ("this DELETE touches ~2.4M rows via Seq Scan"), routing policies can match on `estimated_rows` / `scan_type` (e.g. auto-escalate full-scan DELETEs over 100k rows), and the estimate is folded into the AI analyzer's prompt so risk scoring reflects the actual blast radius. Engines without a plan concept degrade gracefully to an "estimate unavailable" state that never blocks submission. +- **Bytes-scanned cost caps** — on BigQuery, Snowflake and Databricks, where a ten-row result can scan a terabyte, set a maximum bytes-scanned per query on the datasource and, tighter, per grant. A query whose pre-flight scan estimate exceeds the cap is refused before it runs, with a message naming both numbers; the cap is checked again right before execution, so scheduled, recurring, grouped and break-glass runs honour it too. When no estimate exists, the datasource decides explicitly: send the query to a person, or refuse it. For an advisory signal instead, route on `estimated_bytes_scanned` to escalate expensive queries. - **Approval-likelihood prediction (AF-645)** — a triage signal for busy review queues. AccessFlow trains a small per-organization statistical model on your own historical review decisions and shows reviewers the probability that a query gets approved — a bare percentage badge on the review queue, and a labelled "Historical approval likelihood: 78%" card on the query detail page. No LLM call and no external service — it's plain logistic regression over data you already collect (query type, AI risk verdict, cost estimate, time of day, per-submitter and per-datasource approval history), retrained daily. **Advisory only**: it never approves or rejects anything, and it is never an input to routing, grant coverage, or any other decision path. Auto-approved, break-glass, and grant-covered queries are excluded from training because they carry no reviewer judgment, and until an organization has enough decided history for the model to clear its quality gate the UI says "not enough review history yet" rather than showing a number. - **Text-to-query** — opt-in per datasource. Users describe what they want in plain language ("order numbers for the last 5 days") and the AI drafts a schema-grounded query into the editor — in the datasource engine's **native query language** (SQL for relational engines, plus MongoDB shell/JSON, Cypher, CQL, Elasticsearch Query DSL, redis-cli, SQL++, and PartiQL for the NoSQL engines). Reuses the datasource's AI configuration and is grounded in its introspected schema (restricted columns are never referenced). The generated query is only a draft — it's still submitted through the normal pipeline, so AI risk analysis and human review always apply. - **RAG knowledge base** — opt-in per AI configuration. Attach knowledge documents (data-governance policies, naming conventions, schema notes); AccessFlow embeds them with a dedicated embedding model and, at analysis / text-to-SQL time, retrieves the most relevant chunks and injects them into the prompt — so analysis reflects your house rules. Pluggable vector store: in-app **pgvector** (self-contained, auto-provisioned where the DB role permits — and optional: AccessFlow still starts if the extension is absent, with the in-app store disabled) or external **Qdrant**. Retrieval is best-effort and never blocks analysis. diff --git a/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java b/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java index 1a400e155..c191cc949 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java +++ b/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java @@ -54,7 +54,8 @@ void materialize(UUID accessRequestId, UUID approvedByUserId) { return; } // A replaced row's denials carry over (#939): a JIT approval widens capabilities and expiry, - // never lifts a denial an admin set — JIT requests cannot even ask for deny-lists. + // never lifts a denial an admin set — JIT requests cannot even ask for deny-lists. The + // bytes-scanned cap carries over for the same reason (#941): a JIT approval never widens cost. var replaced = replaceExistingTimeBoxedPermission(entity); var command = new CreatePermissionCommand( entity.getRequesterId(), @@ -63,6 +64,7 @@ void materialize(UUID accessRequestId, UUID approvedByUserId) { entity.isCanDdl(), false, null, + replaced.map(DatasourceUserPermissionView::bytesScannedLimitOverride).orElse(null), toList(entity.getAllowedSchemas()), toList(entity.getAllowedTables()), null, diff --git a/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java b/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java index 7f91db21b..94473aaa9 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java @@ -8,6 +8,7 @@ import com.bablsoft.accessflow.ai.api.AiBudgetExceededException; import com.bablsoft.accessflow.ai.api.AiRateLimitExceededException; import com.bablsoft.accessflow.ai.internal.persistence.repo.AiConfigRepository; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; import com.bablsoft.accessflow.core.api.AiAnalysisPersistenceService; import com.bablsoft.accessflow.core.api.AiProviderType; import com.bablsoft.accessflow.core.api.DataClassification; @@ -296,6 +297,14 @@ private String buildCostEstimateContext(UUID queryRequestId) { } sb.append("Plan root operation: ").append(estimate.scanType()).append('.'); } + if (estimate.estimatedBytesScanned() != null) { + if (!sb.isEmpty()) { + sb.append(' '); + } + sb.append("The warehouse estimates this query will scan ") + .append(ByteSizeFormat.format(estimate.estimatedBytesScanned())) + .append('.'); + } return sb.isEmpty() ? null : sb.toString(); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java b/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java index 16684e151..05c0cce50 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java @@ -211,6 +211,11 @@ private String toSnapshotJson(DatasourcePermissionView view) { } else { node.putNull("row_limit_override"); } + if (view.bytesScannedLimitOverride() != null) { + node.put("bytes_scanned_limit_override", view.bytesScannedLimitOverride()); + } else { + node.putNull("bytes_scanned_limit_override"); + } putStringArray(node, "allowed_schemas", view.allowedSchemas()); putStringArray(node, "allowed_tables", view.allowedTables()); putStringArray(node, "restricted_columns", view.restrictedColumns()); diff --git a/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java b/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java index 1d19595a8..415bec54b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java +++ b/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java @@ -132,6 +132,14 @@ public enum AuditAction { * when the guard changed the outcome; never for {@code WARN}, never on a rejection. */ SQL_REVIEW_BLOCKED, + /** + * The bytes-scanned cap (#941) changed a query's outcome: refused it when it left + * {@code PENDING_AI} ({@code stage=decision}), turned an automatic approval into human review, + * or failed it just before execution ({@code stage=execution}). System-attributed — null actor, + * {@code trigger=bytes_scanned_cap}; metadata carries {@code limit}, {@code source}, + * {@code estimated_bytes} and {@code outcome}. + */ + QUERY_BYTES_SCANNED_CAP_ENFORCED, ROW_SECURITY_POLICY_CREATED, ROW_SECURITY_POLICY_UPDATED, ROW_SECURITY_POLICY_DELETED, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java new file mode 100644 index 000000000..1079df1bb --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java @@ -0,0 +1,36 @@ +package com.bablsoft.accessflow.core.api; + +import java.util.Objects; + +/** + * The bytes-scanned cap (#941) binding one user on one datasource: the most restrictive of the + * datasource's {@code max_bytes_scanned_per_query} and the user's merged grant override, together + * with the datasource's missing-estimate policy. + */ +public record AppliedBytesCap(long limit, BytesScannedCapSource source, + BytesCapMissingEstimateAction missingEstimate) { + + public AppliedBytesCap { + if (limit <= 0) { + throw new IllegalArgumentException("limit must be positive"); + } + Objects.requireNonNull(source, "source"); + missingEstimate = missingEstimate == null + ? BytesCapMissingEstimateAction.REQUIRE_REVIEW : missingEstimate; + } + + /** + * Compares an estimate against the cap. {@code null} means no estimate exists; the + * datasource's missing-estimate policy then decides. + */ + public BytesScannedCapOutcome check(Long estimatedBytesScanned) { + if (estimatedBytesScanned == null) { + return missingEstimate == BytesCapMissingEstimateAction.REJECT + ? BytesScannedCapOutcome.NO_ESTIMATE_REJECTED + : BytesScannedCapOutcome.NO_ESTIMATE_REVIEW; + } + return estimatedBytesScanned > limit + ? BytesScannedCapOutcome.EXCEEDED + : BytesScannedCapOutcome.WITHIN; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java new file mode 100644 index 000000000..74f452963 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java @@ -0,0 +1,32 @@ +package com.bablsoft.accessflow.core.api; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** + * Renders a raw byte count for a user-facing message, in decimal units (the unit warehouses bill + * in), keeping the exact figure alongside so a value just over a limit never reads as equal to it: + * {@code 1.5 TB (1500000000000 B)} — the unit symbols need no translation inside a localized message. Mirrors the frontend's {@code formatBytes}. + */ +public final class ByteSizeFormat { + + private static final String[] UNITS = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; + + private ByteSizeFormat() { + } + + public static String format(long bytes) { + if (bytes < 1000) { + return bytes + " B"; + } + var scaled = BigDecimal.valueOf(bytes); + var thousand = BigDecimal.valueOf(1000); + int unit = 0; + while (scaled.compareTo(thousand) >= 0 && unit < UNITS.length - 1) { + scaled = scaled.divide(thousand); + unit++; + } + var rounded = scaled.setScale(2, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + return rounded + " " + UNITS[unit] + " (" + bytes + " B)"; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java new file mode 100644 index 000000000..6269b6e99 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java @@ -0,0 +1,12 @@ +package com.bablsoft.accessflow.core.api; + +/** + * What a bytes-scanned cap (#941) does when the query has no bytes estimate to compare against — + * the estimate failed, the statement shape has no plan (DDL), or the engine returned no bytes + * figure. Never "allow silently": {@link #REQUIRE_REVIEW} suppresses every automatic approval so a + * person decides, {@link #REJECT} refuses the query outright. + */ +public enum BytesCapMissingEstimateAction { + REQUIRE_REVIEW, + REJECT +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java new file mode 100644 index 000000000..2e4f18d3c --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java @@ -0,0 +1,34 @@ +package com.bablsoft.accessflow.core.api; + +/** + * The bytes-scanned cap (#941) refused a statement just before execution — its pre-flight + * estimate exceeds the cap, or it has none and the datasource rejects on a missing estimate. The + * message is the caller's localized explanation naming the estimate and the limit; execution paths + * record it as the failure reason. + */ +public class BytesScannedCapExceededException extends RuntimeException { + + private final transient AppliedBytesCap cap; + private final Long estimatedBytes; + private final BytesScannedCapOutcome outcome; + + public BytesScannedCapExceededException(String message, AppliedBytesCap cap, + Long estimatedBytes, BytesScannedCapOutcome outcome) { + super(message); + this.cap = cap; + this.estimatedBytes = estimatedBytes; + this.outcome = outcome; + } + + public AppliedBytesCap cap() { + return cap; + } + + public Long estimatedBytes() { + return estimatedBytes; + } + + public BytesScannedCapOutcome outcome() { + return outcome; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java new file mode 100644 index 000000000..4bbfe770a --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java @@ -0,0 +1,19 @@ +package com.bablsoft.accessflow.core.api; + +/** + * A bytes-scanned cap (#941) — on a datasource or on a grant — was configured for an engine whose + * dry-run does not report a bytes estimate. See {@link BytesScannedCapSupport}. + */ +public final class BytesScannedCapNotSupportedException extends DatasourceAdminException { + + private final DbType dbType; + + public BytesScannedCapNotSupportedException(DbType dbType) { + super("bytes-scanned caps are not supported for db_type " + dbType); + this.dbType = dbType; + } + + public DbType dbType() { + return dbType; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java new file mode 100644 index 000000000..1845463e1 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java @@ -0,0 +1,18 @@ +package com.bablsoft.accessflow.core.api; + +/** How a query's bytes estimate compared against the cap that applied to it (#941). */ +public enum BytesScannedCapOutcome { + /** An estimate existed and was at or below the cap. */ + WITHIN, + /** The estimate was above the cap: the query is refused. */ + EXCEEDED, + /** No estimate, and the datasource says {@link BytesCapMissingEstimateAction#REQUIRE_REVIEW}. */ + NO_ESTIMATE_REVIEW, + /** No estimate, and the datasource says {@link BytesCapMissingEstimateAction#REJECT}. */ + NO_ESTIMATE_REJECTED; + + /** Whether this outcome refuses the query. */ + public boolean rejects() { + return this == EXCEEDED || this == NO_ESTIMATE_REJECTED; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java new file mode 100644 index 000000000..172037384 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java @@ -0,0 +1,13 @@ +package com.bablsoft.accessflow.core.api; + +import java.util.Optional; +import java.util.UUID; + +/** + * Resolves the bytes-scanned cap (#941) that binds a user on a datasource. Empty when neither the + * datasource nor any of the user's grants sets one. + */ +public interface BytesScannedCapResolutionService { + + Optional resolve(UUID datasourceId, UUID userId); +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java new file mode 100644 index 000000000..09c787239 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java @@ -0,0 +1,9 @@ +package com.bablsoft.accessflow.core.api; + +/** Which configuration supplied the binding bytes-scanned cap (#941). */ +public enum BytesScannedCapSource { + /** {@code datasources.max_bytes_scanned_per_query}. */ + DATASOURCE, + /** A {@code bytes_scanned_limit_override} on the submitter's direct or group grant. */ + GRANT +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java new file mode 100644 index 000000000..14c8a9477 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java @@ -0,0 +1,26 @@ +package com.bablsoft.accessflow.core.api; + +import java.util.Set; + +/** + * The engines whose pre-flight dry-run reports a bytes-scanned estimate (AF-634), and therefore + * the only ones a bytes-scanned cap (#941) may be configured on. On any other engine every query + * would have no estimate, so a cap there would either reject everything or be a no-op — both + * surprising — and configuring one is refused instead. + */ +public final class BytesScannedCapSupport { + + private static final Set SUPPORTED = Set.of(DbType.BIGQUERY, DbType.SNOWFLAKE, + DbType.DATABRICKS); + + private BytesScannedCapSupport() { + } + + public static boolean supports(DbType dbType) { + return dbType != null && SUPPORTED.contains(dbType); + } + + public static Set supportedTypes() { + return SUPPORTED; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java index c2315faa3..3ef551745 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java @@ -30,8 +30,32 @@ public record CreateDatasourceCommand( Boolean resultCacheEnabled, Integer resultCacheTtlSeconds, String privateKeyPassphrase, - DatasourceEnvironment environment + DatasourceEnvironment environment, + Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { + /** + * Backward-compatible constructor for the pre-#941 canonical shape (no bytes-scanned cap); + * delegates with {@code null} — no cap, default missing-estimate policy. + */ + public CreateDatasourceCommand( + UUID organizationId, String name, DbType dbType, String host, Integer port, + String databaseName, String username, String password, SslMode sslMode, + Integer connectionPoolSize, Integer maxRowsPerQuery, Boolean requireReviewReads, + Boolean requireReviewWrites, UUID reviewPlanId, Boolean aiAnalysisEnabled, + UUID aiConfigId, Boolean textToSqlEnabled, UUID customDriverId, String connectorId, + String jdbcUrlOverride, List readReplicas, + String localDatacenter, String apiKey, Boolean resultCacheEnabled, + Integer resultCacheTtlSeconds, String privateKeyPassphrase, + DatasourceEnvironment environment) { + this(organizationId, name, dbType, host, port, databaseName, username, password, sslMode, + connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, + reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, + connectorId, jdbcUrlOverride, readReplicas, localDatacenter, apiKey, + resultCacheEnabled, resultCacheTtlSeconds, privateKeyPassphrase, environment, + null, null); + } + /** * Backward-compatible constructor for the pre-#861 canonical shape (no {@code environment}); * delegates with {@code null} — an unset environment is a legitimate state. @@ -49,7 +73,8 @@ public CreateDatasourceCommand( connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, connectorId, jdbcUrlOverride, readReplicas, localDatacenter, apiKey, - resultCacheEnabled, resultCacheTtlSeconds, privateKeyPassphrase, null); + resultCacheEnabled, resultCacheTtlSeconds, privateKeyPassphrase, + (DatasourceEnvironment) null); } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java index dafda393d..d6ec918ec 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java @@ -11,6 +11,7 @@ public record CreateDatasourceGroupPermissionCommand( Boolean canDdl, Boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java index 242e6c9d7..f405ae08c 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java @@ -15,6 +15,7 @@ public record CreatePermissionCommand( Boolean canDdl, Boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java index 5478b9a0b..85851d0eb 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java @@ -10,6 +10,7 @@ public sealed class DatasourceAdminException extends RuntimeException IllegalDatasourcePermissionException, DeniedColumnsNotSupportedException, DeniedShapesNotSupportedException, + BytesScannedCapNotSupportedException, MissingAiConfigForDatasourceException, TableNotFoundException { diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java index 45c8f71e6..14c6dd307 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java @@ -15,6 +15,7 @@ public record DatasourceGroupPermissionView( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java index 93fbafec0..5005ed900 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java @@ -17,6 +17,8 @@ * row id * @param groupId the granting group, or {@code null} for a direct grant * @param rowLimitOverride this grant's row cap, or {@code null} when it sets none (#933) + * @param bytesScannedLimitOverride this grant's bytes-scanned cap, or {@code null} when it sets + * none (#941) * @param expiresAt {@code null} means this contribution never expires * @param accessGrantRequestId the JIT {@code access_grant_request} a direct row materialises * (#969); {@code null} on an admin-created row and always on a @@ -41,9 +43,24 @@ public record DatasourcePermissionContribution( List deniedTables, List deniedShapes, Integer rowLimitOverride, + Long bytesScannedLimitOverride, Instant expiresAt, UUID accessGrantRequestId) { + /** Backward-compatible constructor without the #941 bytes-scanned cap override. */ + public DatasourcePermissionContribution( + DatasourcePermissionSourceKind sourceKind, UUID sourceId, UUID userId, + UUID datasourceId, UUID groupId, String groupName, boolean canRead, boolean canWrite, + boolean canDdl, boolean canBreakGlass, List allowedSchemas, + List allowedTables, List restrictedColumns, List deniedColumns, + List deniedSchemas, List deniedTables, List deniedShapes, + Integer rowLimitOverride, Instant expiresAt, UUID accessGrantRequestId) { + this(sourceKind, sourceId, userId, datasourceId, groupId, groupName, canRead, canWrite, + canDdl, canBreakGlass, allowedSchemas, allowedTables, restrictedColumns, + deniedColumns, deniedSchemas, deniedTables, deniedShapes, rowLimitOverride, null, + expiresAt, accessGrantRequestId); + } + public DatasourcePermissionContribution { allowedSchemas = allowedSchemas == null ? List.of() : List.copyOf(allowedSchemas); allowedTables = allowedTables == null ? List.of() : List.copyOf(allowedTables); diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java index 6b8a17a5b..cdf278f54 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java @@ -15,6 +15,7 @@ public record DatasourcePermissionView( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java index be019dec1..a9a6e6d68 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java @@ -20,5 +20,19 @@ public record DatasourceUserPermissionView( List deniedTables, List deniedShapes, Integer rowLimitOverride, + Long bytesScannedLimitOverride, Instant expiresAt) { + + /** Backward-compatible constructor without the #941 bytes-scanned cap override. */ + public DatasourceUserPermissionView(UUID id, UUID userId, UUID datasourceId, boolean canRead, + boolean canWrite, boolean canDdl, boolean canBreakGlass, + List allowedSchemas, List allowedTables, + List restrictedColumns, List deniedColumns, + List deniedSchemas, List deniedTables, + List deniedShapes, Integer rowLimitOverride, + Instant expiresAt) { + this(id, userId, datasourceId, canRead, canWrite, canDdl, canBreakGlass, allowedSchemas, + allowedTables, restrictedColumns, deniedColumns, deniedSchemas, deniedTables, + deniedShapes, rowLimitOverride, null, expiresAt); + } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java index b579fd559..a7ef06597 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java @@ -31,7 +31,9 @@ public record DatasourceView( String localDatacenter, boolean resultCacheEnabled, Integer resultCacheTtlSeconds, - DatasourceEnvironment environment + DatasourceEnvironment environment, + Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { /** One read-replica endpoint as exposed to admins — never carries the password. */ public record ReadReplicaView(UUID id, String jdbcUrl, String username) { @@ -39,6 +41,28 @@ public record ReadReplicaView(UUID id, String jdbcUrl, String username) { public DatasourceView { readReplicas = readReplicas == null ? List.of() : List.copyOf(readReplicas); + bytesCapMissingEstimate = bytesCapMissingEstimate == null + ? BytesCapMissingEstimateAction.REQUIRE_REVIEW : bytesCapMissingEstimate; + } + + /** + * Backward-compatible constructor for the pre-#941 canonical shape (no bytes-scanned cap); + * delegates with no cap and the default missing-estimate policy. + */ + public DatasourceView( + UUID id, UUID organizationId, String name, DbType dbType, String host, Integer port, + String databaseName, String username, SslMode sslMode, int connectionPoolSize, + int maxRowsPerQuery, boolean requireReviewReads, boolean requireReviewWrites, + UUID reviewPlanId, boolean aiAnalysisEnabled, UUID aiConfigId, boolean textToSqlEnabled, + UUID customDriverId, String connectorId, String jdbcUrlOverride, + List readReplicas, boolean active, Instant createdAt, + String localDatacenter, boolean resultCacheEnabled, Integer resultCacheTtlSeconds, + DatasourceEnvironment environment) { + this(id, organizationId, name, dbType, host, port, databaseName, username, sslMode, + connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, + reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, + connectorId, jdbcUrlOverride, readReplicas, active, createdAt, localDatacenter, + resultCacheEnabled, resultCacheTtlSeconds, environment, null, null); } /** @@ -57,7 +81,7 @@ public DatasourceView( connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, connectorId, jdbcUrlOverride, readReplicas, active, createdAt, localDatacenter, - resultCacheEnabled, resultCacheTtlSeconds, null); + resultCacheEnabled, resultCacheTtlSeconds, (DatasourceEnvironment) null); } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java index 615396324..98851f9bb 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java @@ -3,7 +3,8 @@ /** * Everything needed to persist one {@code query_estimates} row (issue AF-624). Mirrors * {@link PersistAiAnalysisCommand}: the proxy module computes the estimate and hands the values to - * {@link QueryEstimatePersistenceService}. All value fields are nullable best-effort signals. + * {@link QueryEstimatePersistenceService}. All value fields are nullable best-effort signals; + * {@code estimatedBytesScanned} is the warehouse engines' native scan estimate in raw bytes (#941). */ public record PersistQueryEstimateCommand( String engineId, @@ -13,6 +14,7 @@ public record PersistQueryEstimateCommand( Long affectedRowCount, String scanType, Double estimatedCost, + Long estimatedBytesScanned, String planJson, String rawPlan, String unsupportedReason, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java index 9f799408b..163b18db8 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java @@ -49,7 +49,40 @@ public record QueryDetailView( String onBehalfOfEmail, /** The calling application (#938) and how it was learned; both null when unknown. */ String applicationName, - ApplicationNameSource applicationNameSource) { + ApplicationNameSource applicationNameSource, + /** The bytes-scanned cap (#941) that applied when the query left PENDING_AI; null when none. */ + BytesScannedCapDetail bytesScannedCap) { + + /** Backward-compatible constructor without the #941 bytes-scanned cap. */ + public QueryDetailView(UUID id, UUID datasourceId, String datasourceName, DbType dbType, + UUID organizationId, UUID submittedByUserId, String submittedByEmail, + String submittedByDisplayName, String sqlText, QueryType queryType, + QueryStatus status, String justification, AiAnalysisDetail aiAnalysis, + CostEstimateDetail costEstimate, + ApprovalPredictionDetail approvalPrediction, Long rowsAffected, + Integer durationMs, String errorMessage, UUID previousRunId, + UUID approvedByGrantId, String reviewPlanName, + Integer approvalTimeoutHours, Instant escalatedAt, + Integer escalationAfterHours, List reviewDecisions, + Instant scheduledFor, String recurrenceRule, Instant recurrenceUntil, + Instant recurrenceNextRunAt, String recurrenceHaltedReason, + UUID recurringParentId, Instant createdAt, Instant updatedAt, + UUID onBehalfOfUserId, String onBehalfOfEmail, String applicationName, + ApplicationNameSource applicationNameSource) { + this(id, datasourceId, datasourceName, dbType, organizationId, submittedByUserId, + submittedByEmail, submittedByDisplayName, sqlText, queryType, status, justification, + aiAnalysis, costEstimate, approvalPrediction, rowsAffected, durationMs, + errorMessage, previousRunId, approvedByGrantId, reviewPlanName, + approvalTimeoutHours, escalatedAt, escalationAfterHours, reviewDecisions, + scheduledFor, recurrenceRule, recurrenceUntil, recurrenceNextRunAt, + recurrenceHaltedReason, recurringParentId, createdAt, updatedAt, onBehalfOfUserId, + onBehalfOfEmail, applicationName, applicationNameSource, null); + } + + /** The bytes-scanned cap (#941) recorded on the request, and how its estimate compared. */ + public record BytesScannedCapDetail(long limit, BytesScannedCapSource source, + BytesScannedCapOutcome outcome) { + } /** Backward-compatible constructor without the #938 calling application. */ public QueryDetailView(UUID id, UUID datasourceId, String datasourceName, DbType dbType, @@ -73,7 +106,7 @@ public QueryDetailView(UUID id, UUID datasourceId, String datasourceName, DbType approvalTimeoutHours, escalatedAt, escalationAfterHours, reviewDecisions, scheduledFor, recurrenceRule, recurrenceUntil, recurrenceNextRunAt, recurrenceHaltedReason, recurringParentId, createdAt, updatedAt, onBehalfOfUserId, - onBehalfOfEmail, null, null); + onBehalfOfEmail, (String) null, (ApplicationNameSource) null); } /** Backward-compatible constructor without the #874 on-behalf-of principal. */ @@ -186,7 +219,20 @@ public record CostEstimateDetail( String unsupportedReason, boolean failed, String errorMessage, - Integer durationMs) { + Integer durationMs, + /** The warehouse engines' pre-flight scan estimate in raw bytes (#941); else null. */ + Long estimatedBytesScanned) { + + /** Backward-compatible constructor without the #941 bytes estimate. */ + public CostEstimateDetail(UUID id, String engineId, QueryType queryType, boolean supported, + Long estimatedRows, Long affectedRowCount, String scanType, + Double estimatedCost, String planJson, String rawPlan, + String unsupportedReason, boolean failed, String errorMessage, + Integer durationMs) { + this(id, engineId, queryType, supported, estimatedRows, affectedRowCount, scanType, + estimatedCost, planJson, rawPlan, unsupportedReason, failed, errorMessage, + durationMs, null); + } } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java index bd118dddb..9d282baf5 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java @@ -10,7 +10,8 @@ * is {@code false} when the engine has no plan concept; {@code failed} marks an unexpected * computation error. {@code estimatedRows}, {@code affectedRowCount}, {@code scanType} and * {@code estimatedCost} are individually nullable — best-effort signals filled from what the - * engine's plan exposes. + * engine's plan exposes. {@code estimatedBytesScanned} (#941) is the warehouse engines' native + * pre-flight scan estimate in raw bytes, null for every engine that does not report one. */ public record QueryEstimateSnapshot( UUID id, @@ -22,6 +23,7 @@ public record QueryEstimateSnapshot( Long affectedRowCount, String scanType, Double estimatedCost, + Long estimatedBytesScanned, String planJson, String rawPlan, String unsupportedReason, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java index 690bde05a..826e2d896 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java @@ -25,6 +25,14 @@ public interface QueryRequestStateService { */ void approveByAccessGrant(UUID queryRequestId, UUID accessGrantId); + /** + * Records the bytes-scanned cap (#941) that applied when the query left {@code PENDING_AI}, + * and how its estimate compared. A plain stamp, not a transition — it runs just before the + * decision is applied so the detail view can explain the outcome. + */ + void recordBytesScannedCap(UUID queryRequestId, long limit, BytesScannedCapSource source, + BytesScannedCapOutcome outcome); + /** * Inserts an {@code APPROVED} {@link com.bablsoft.accessflow.core.api.DecisionType} row * for the given reviewer/stage and, if the per-stage threshold is now met AND it was the diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java index 020fa65b2..ece064003 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java @@ -10,7 +10,8 @@ * {@code readReplicas} (AF-457), {@code null} keeps the current endpoint list, an empty list * deletes all endpoints, and a non-empty list is a full replacement merged by endpoint id (items * with a known {@code id} update that row; items without create new rows; stored rows absent from - * the list are removed). + * the list are removed). The bytes-scanned cap (#941) follows the same shape as the environment: + * {@code clearMaxBytesScannedPerQuery=true} removes the cap, a non-null value wins when both are sent. */ public record UpdateDatasourceCommand( String name, @@ -38,8 +39,33 @@ public record UpdateDatasourceCommand( Integer resultCacheTtlSeconds, String privateKeyPassphrase, DatasourceEnvironment environment, - Boolean clearEnvironment + Boolean clearEnvironment, + Long maxBytesScannedPerQuery, + Boolean clearMaxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { + /** + * Backward-compatible constructor for the pre-#941 canonical shape (no bytes-scanned cap); + * delegates with {@code null} — the cap and its policy stay unchanged. + */ + public UpdateDatasourceCommand( + String name, String host, Integer port, String databaseName, String username, + String password, SslMode sslMode, Integer connectionPoolSize, Integer maxRowsPerQuery, + Boolean requireReviewReads, Boolean requireReviewWrites, UUID reviewPlanId, + Boolean aiAnalysisEnabled, UUID aiConfigId, Boolean textToSqlEnabled, + Boolean clearAiConfig, String jdbcUrlOverride, + List readReplicas, Boolean active, String localDatacenter, + String apiKey, Boolean resultCacheEnabled, Integer resultCacheTtlSeconds, + String privateKeyPassphrase, DatasourceEnvironment environment, + Boolean clearEnvironment) { + this(name, host, port, databaseName, username, password, sslMode, connectionPoolSize, + maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, + aiAnalysisEnabled, aiConfigId, textToSqlEnabled, clearAiConfig, jdbcUrlOverride, + readReplicas, active, localDatacenter, apiKey, resultCacheEnabled, + resultCacheTtlSeconds, privateKeyPassphrase, environment, clearEnvironment, + null, null, null); + } + /** * Backward-compatible constructor for the pre-#861 canonical shape (no {@code environment} / * {@code clearEnvironment}); delegates with {@code null} — the environment stays unchanged. @@ -57,7 +83,8 @@ public UpdateDatasourceCommand( maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, clearAiConfig, jdbcUrlOverride, readReplicas, active, localDatacenter, apiKey, resultCacheEnabled, - resultCacheTtlSeconds, privateKeyPassphrase, null, null); + resultCacheTtlSeconds, privateKeyPassphrase, (DatasourceEnvironment) null, + (Boolean) null); } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java b/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java index bc0d8cbf3..5da8a9581 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java @@ -3,9 +3,10 @@ import java.util.UUID; /** - * Published when a query is auto-rejected by the state machine because a routing policy matched with - * the {@code AUTO_REJECT} action. Carries the matched {@code routing_policy.id} and the reason so the - * audit and notification listeners can record provenance. + * Published when a query is rejected without a reviewer: a routing policy matched with the + * {@code AUTO_REJECT} action (carrying its {@code routing_policy.id}), the bytes-scanned cap refused + * it (#941), or an external ticket resolution rejected it — the last two with a null policy id. The + * reason lets the audit and notification listeners record provenance. */ public record QueryAutoRejectedEvent(UUID queryRequestId, UUID matchedPolicyId, String reason) { } diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java index bfb9f64b4..81aa1dd9d 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java @@ -21,6 +21,8 @@ import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.DeniedColumns; import com.bablsoft.accessflow.core.api.DeniedColumnsNotSupportedException; +import com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException; +import com.bablsoft.accessflow.core.api.BytesScannedCapSupport; import com.bablsoft.accessflow.core.api.DeniedShapes; import com.bablsoft.accessflow.core.api.DeniedShapesNotSupportedException; import com.bablsoft.accessflow.core.api.DeniedTables; @@ -249,6 +251,11 @@ public DatasourceView create(CreateDatasourceCommand command) { entity.setResultCacheTtlSeconds(command.resultCacheTtlSeconds()); } entity.setEnvironment(command.environment()); + entity.setMaxBytesScannedPerQuery(bytesCap(entity.getDbType(), + command.maxBytesScannedPerQuery())); + if (command.bytesCapMissingEstimate() != null) { + entity.setBytesCapMissingEstimate(command.bytesCapMissingEstimate()); + } entity.setActive(true); return toView(datasourceRepository.save(entity)); } @@ -360,6 +367,17 @@ public DatasourceView update(UUID id, UUID organizationId, UpdateDatasourceComma if (command.environment() != null) { entity.setEnvironment(command.environment()); } + // #941: same null / clear / value shape as the environment. + if (Boolean.TRUE.equals(command.clearMaxBytesScannedPerQuery())) { + entity.setMaxBytesScannedPerQuery(null); + } + if (command.maxBytesScannedPerQuery() != null) { + entity.setMaxBytesScannedPerQuery(bytesCap(entity.getDbType(), + command.maxBytesScannedPerQuery())); + } + if (command.bytesCapMissingEstimate() != null) { + entity.setBytesCapMissingEstimate(command.bytesCapMissingEstimate()); + } if (command.active() != null) { entity.setActive(command.active()); } @@ -654,6 +672,8 @@ public DatasourcePermissionView grantPermission(UUID datasourceId, UUID organiza entity.setCanDdl(Boolean.TRUE.equals(command.canDdl())); entity.setCanBreakGlass(Boolean.TRUE.equals(command.canBreakGlass())); entity.setRowLimitOverride(command.rowLimitOverride()); + entity.setBytesScannedLimitOverride(bytesCap(datasource.getDbType(), + command.bytesScannedLimitOverride())); entity.setAllowedSchemas(toArray(command.allowedSchemas())); entity.setAllowedTables(toArray(command.allowedTables())); entity.setRestrictedColumns(toArray(command.restrictedColumns())); @@ -714,6 +734,8 @@ public DatasourceGroupPermissionView grantGroupPermission( entity.setCanDdl(Boolean.TRUE.equals(command.canDdl())); entity.setCanBreakGlass(Boolean.TRUE.equals(command.canBreakGlass())); entity.setRowLimitOverride(command.rowLimitOverride()); + entity.setBytesScannedLimitOverride(bytesCap(datasource.getDbType(), + command.bytesScannedLimitOverride())); entity.setAllowedSchemas(toArray(command.allowedSchemas())); entity.setAllowedTables(toArray(command.allowedTables())); entity.setRestrictedColumns(toArray(command.restrictedColumns())); @@ -780,7 +802,9 @@ private DatasourceView toView(DatasourceEntity entity) { entity.getLocalDatacenter(), entity.isResultCacheEnabled(), entity.getResultCacheTtlSeconds(), - entity.getEnvironment()); + entity.getEnvironment(), + entity.getMaxBytesScannedPerQuery(), + entity.getBytesCapMissingEstimate()); } private CustomJdbcDriverEntity resolveCustomDriverForCreate(CreateDatasourceCommand command) { @@ -1150,6 +1174,17 @@ private String[] deniedShapes(DatasourceEntity datasource, List raw) return names; } + /** + * A bytes-scanned cap (#941) is refused on an engine that reports no bytes estimate — there it + * could only reject everything or nothing. Null passes through (no cap). + */ + private static Long bytesCap(DbType dbType, Long cap) { + if (cap != null && !BytesScannedCapSupport.supports(dbType)) { + throw new BytesScannedCapNotSupportedException(dbType); + } + return cap; + } + private DatasourcePermissionView toPermissionView(DatasourceUserPermissionEntity entity) { UserEntity user = entity.getUser(); return new DatasourcePermissionView( @@ -1163,6 +1198,7 @@ private DatasourcePermissionView toPermissionView(DatasourceUserPermissionEntity entity.isCanDdl(), entity.isCanBreakGlass(), entity.getRowLimitOverride(), + entity.getBytesScannedLimitOverride(), toList(entity.getAllowedSchemas()), toList(entity.getAllowedTables()), toList(entity.getRestrictedColumns()), @@ -1189,6 +1225,7 @@ private DatasourceGroupPermissionView toGroupPermissionView( entity.isCanDdl(), entity.isCanBreakGlass(), entity.getRowLimitOverride(), + entity.getBytesScannedLimitOverride(), toList(entity.getAllowedSchemas()), toList(entity.getAllowedTables()), toList(entity.getRestrictedColumns()), diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java new file mode 100644 index 000000000..c069a6cf5 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java @@ -0,0 +1,49 @@ +package com.bablsoft.accessflow.core.internal; + +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionLookupService; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionView; +import com.bablsoft.accessflow.core.internal.persistence.repo.DatasourceRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Optional; +import java.util.UUID; + +/** + * The most restrictive of the datasource's {@code max_bytes_scanned_per_query} and the user's merged + * grant override (#941). On a tie the datasource is named as the source — it is the setting an + * admin would have to change for the limit to move. + */ +@Service +@RequiredArgsConstructor +class DefaultBytesScannedCapResolutionService implements BytesScannedCapResolutionService { + + private final DatasourceRepository datasourceRepository; + private final DatasourceUserPermissionLookupService permissionLookupService; + + @Override + @Transactional(readOnly = true) + public Optional resolve(UUID datasourceId, UUID userId) { + var datasource = datasourceRepository.findById(datasourceId).orElse(null); + if (datasource == null) { + return Optional.empty(); + } + var datasourceCap = datasource.getMaxBytesScannedPerQuery(); + var grantCap = userId == null ? null : permissionLookupService.findFor(userId, datasourceId) + .map(DatasourceUserPermissionView::bytesScannedLimitOverride) + .orElse(null); + if (datasourceCap == null && grantCap == null) { + return Optional.empty(); + } + var missing = datasource.getBytesCapMissingEstimate(); + if (grantCap != null && (datasourceCap == null || grantCap < datasourceCap)) { + return Optional.of(new AppliedBytesCap(grantCap, BytesScannedCapSource.GRANT, missing)); + } + return Optional.of(new AppliedBytesCap(datasourceCap, BytesScannedCapSource.DATASOURCE, + missing)); + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java index d2864909b..e0100f78d 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java @@ -215,6 +215,7 @@ private static DatasourceUserPermissionView merge(UUID userId, UUID datasourceId unionDenied(parts, DatasourcePermissionContribution::deniedTables), unionDeniedShapes(parts), minRowLimit(parts), + minBytesLimit(parts), anyNeverExpires ? null : expiresAt); } @@ -254,6 +255,18 @@ private static Integer minRowLimit(List parts) return min; } + /** Like {@link #minRowLimit}: the most restrictive bytes-scanned cap wins (#941). */ + private static Long minBytesLimit(List parts) { + Long min = null; + for (var p : parts) { + var limit = p.bytesScannedLimitOverride(); + if (limit != null && (min == null || limit < min)) { + min = limit; + } + } + return min; + } + /** Allow-list union: a null/empty contribution means "all allowed", so it wins → empty list. */ private static List unionAllowList( List parts, @@ -318,6 +331,7 @@ private static DatasourceUserPermissionView toDirectView(DatasourceUserPermissio DeniedTables.normalize(toList(entity.getDeniedTables())), DeniedShapes.fromNames(toList(entity.getDeniedShapes())), entity.getRowLimitOverride(), + entity.getBytesScannedLimitOverride(), entity.getExpiresAt()); } @@ -330,7 +344,8 @@ private static DatasourcePermissionContribution toContribution( toList(e.getRestrictedColumns()), toList(e.getDeniedColumns()), toList(e.getDeniedSchemas()), toList(e.getDeniedTables()), DeniedShapes.fromNames(toList(e.getDeniedShapes())), - e.getRowLimitOverride(), e.getExpiresAt(), e.getAccessGrantRequestId()); + e.getRowLimitOverride(), e.getBytesScannedLimitOverride(), e.getExpiresAt(), + e.getAccessGrantRequestId()); } private static DatasourcePermissionContribution toContribution( @@ -342,7 +357,8 @@ private static DatasourcePermissionContribution toContribution( toList(e.getRestrictedColumns()), toList(e.getDeniedColumns()), toList(e.getDeniedSchemas()), toList(e.getDeniedTables()), DeniedShapes.fromNames(toList(e.getDeniedShapes())), - e.getRowLimitOverride(), e.getExpiresAt(), null); + e.getRowLimitOverride(), e.getBytesScannedLimitOverride(), e.getExpiresAt(), + null); } private static List toList(String[] array) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java index 642303f49..40c809d5f 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java @@ -43,6 +43,7 @@ public UUID persist(UUID queryRequestId, PersistQueryEstimateCommand command) { entity.setAffectedRowCount(command.affectedRowCount()); entity.setScanType(command.scanType()); entity.setEstimatedCost(command.estimatedCost()); + entity.setEstimatedBytesScanned(command.estimatedBytesScanned()); entity.setPlan(command.planJson()); entity.setRawPlan(command.rawPlan()); entity.setUnsupportedReason(command.unsupportedReason()); @@ -72,6 +73,7 @@ private QueryEstimateSnapshot toSnapshot(QueryEstimateEntity entity) { entity.getAffectedRowCount(), entity.getScanType(), entity.getEstimatedCost(), + entity.getEstimatedBytesScanned(), entity.getPlan(), entity.getRawPlan(), entity.getUnsupportedReason(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java index 5c1892425..3b2271e1f 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java @@ -362,7 +362,13 @@ private QueryDetailView toDetailView(QueryRequestEntity entity) { entity.getOnBehalfOfUserId(), onBehalfOfEmail(entity.getOnBehalfOfUserId()), entity.getApplicationName(), - entity.getApplicationNameSource()); + entity.getApplicationNameSource(), + entity.getBytesScannedCap() != null && entity.getBytesScannedCapSource() != null + && entity.getBytesScannedCapOutcome() != null + ? new QueryDetailView.BytesScannedCapDetail(entity.getBytesScannedCap(), + entity.getBytesScannedCapSource(), + entity.getBytesScannedCapOutcome()) + : null); } private String onBehalfOfEmail(UUID onBehalfOfUserId) { @@ -444,7 +450,8 @@ private static QueryDetailView.CostEstimateDetail toCostEstimateDetail( entity.getUnsupportedReason(), entity.isFailed(), entity.getErrorMessage(), - entity.getDurationMs()); + entity.getDurationMs(), + entity.getEstimatedBytesScanned()); } private static QueryDetailView.ApprovalPredictionDetail toApprovalPredictionDetail( diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java index 2f47c6be0..30788db44 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java @@ -3,6 +3,8 @@ import com.bablsoft.accessflow.core.api.DecisionType; import com.bablsoft.accessflow.core.api.IllegalQueryStatusTransitionException; import com.bablsoft.accessflow.core.api.QueryRequestNotFoundException; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.QueryRequestStateService; import com.bablsoft.accessflow.core.api.QueryStatus; import com.bablsoft.accessflow.core.api.RecordApprovalCommand; @@ -65,6 +67,18 @@ public void approveByAccessGrant(UUID queryRequestId, UUID accessGrantId) { publishStatusChanged(entity, previous, QueryStatus.APPROVED); } + @Override + @Transactional + public void recordBytesScannedCap(UUID queryRequestId, long limit, + BytesScannedCapSource source, + BytesScannedCapOutcome outcome) { + var entity = lockOrThrow(queryRequestId); + entity.setBytesScannedCap(limit); + entity.setBytesScannedCapSource(source); + entity.setBytesScannedCapOutcome(outcome); + queryRequestRepository.save(entity); + } + @Override @Transactional public RecordDecisionResult recordApprovalAndAdvance(RecordApprovalCommand command) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java index c68d54602..6f4172fd3 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java @@ -1,6 +1,7 @@ package com.bablsoft.accessflow.core.internal.persistence.entity; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.SslMode; @@ -124,6 +125,17 @@ public class DatasourceEntity { @Column(name = "environment", columnDefinition = "datasource_environment") private DatasourceEnvironment environment; + /** #941: pre-flight bytes-scanned cap; null = no cap. Only set on bytes-reporting engines. */ + @Column(name = "max_bytes_scanned_per_query") + private Long maxBytesScannedPerQuery; + + @Enumerated(EnumType.STRING) + @JdbcType(PostgreSQLEnumJdbcType.class) + @Column(name = "bytes_cap_missing_estimate", nullable = false, + columnDefinition = "bytes_cap_missing_estimate_action") + private BytesCapMissingEstimateAction bytesCapMissingEstimate = + BytesCapMissingEstimateAction.REQUIRE_REVIEW; + @JsonIgnore @Column(name = "api_key_encrypted", columnDefinition = "TEXT") private String apiKeyEncrypted; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java index 4c80ac2a0..0cedcdf81 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java @@ -53,6 +53,10 @@ public class DatasourceGroupPermissionEntity { @Column(name = "row_limit_override") private Integer rowLimitOverride; + /** #941: this grant's bytes-scanned cap; null = none. Most restrictive across grants wins. */ + @Column(name = "bytes_scanned_limit_override") + private Long bytesScannedLimitOverride; + @JdbcTypeCode(SqlTypes.ARRAY) @Column(name = "allowed_schemas", columnDefinition = "text[]") private String[] allowedSchemas; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java index d2393fd2c..a9c4c8647 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java @@ -49,6 +49,10 @@ public class DatasourceUserPermissionEntity { @Column(name = "row_limit_override") private Integer rowLimitOverride; + /** #941: this grant's bytes-scanned cap; null = none. Most restrictive across grants wins. */ + @Column(name = "bytes_scanned_limit_override") + private Long bytesScannedLimitOverride; + @JdbcTypeCode(SqlTypes.ARRAY) @Column(name = "allowed_schemas", columnDefinition = "text[]") private String[] allowedSchemas; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java index c59b8f1aa..fd73f6945 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java @@ -58,6 +58,9 @@ public class QueryEstimateEntity { @Column(name = "estimated_cost") private Double estimatedCost; + @Column(name = "estimated_bytes_scanned") + private Long estimatedBytesScanned; + @JdbcTypeCode(SqlTypes.JSON) @Column(columnDefinition = "jsonb") private String plan; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java index 1db2d7440..73d3c690e 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java @@ -1,6 +1,8 @@ package com.bablsoft.accessflow.core.internal.persistence.entity; import com.bablsoft.accessflow.core.api.ApplicationNameSource; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.QueryStatus; import com.bablsoft.accessflow.core.api.QueryType; import com.bablsoft.accessflow.core.api.SubmissionReason; @@ -152,6 +154,21 @@ public class QueryRequestEntity { @Column(name = "application_name_source", columnDefinition = "application_name_source") private ApplicationNameSource applicationNameSource; + // The bytes-scanned cap (#941) that applied when the query left PENDING_AI — stamped by the + // workflow so the detail view can say why a query was refused or forced to review. + @Column(name = "bytes_scanned_cap") + private Long bytesScannedCap; + + @Enumerated(EnumType.STRING) + @JdbcType(PostgreSQLEnumJdbcType.class) + @Column(name = "bytes_scanned_cap_source", columnDefinition = "bytes_scanned_cap_source") + private BytesScannedCapSource bytesScannedCapSource; + + @Enumerated(EnumType.STRING) + @JdbcType(PostgreSQLEnumJdbcType.class) + @Column(name = "bytes_scanned_cap_outcome", columnDefinition = "bytes_scanned_cap_outcome") + private BytesScannedCapOutcome bytesScannedCapOutcome; + @Version @Column(name = "updated_at", nullable = false) private Instant updatedAt = Instant.now(); diff --git a/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java b/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java index e93c831d5..7ae79fe13 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java @@ -1,6 +1,7 @@ package com.bablsoft.accessflow.proxy.api; import com.bablsoft.accessflow.core.api.QueryEstimateSnapshot; +import com.bablsoft.accessflow.core.api.QueryExecutionRequest; import java.util.Optional; import java.util.UUID; @@ -22,4 +23,13 @@ public interface QueryCostEstimateService { * caller. Empty only when the query request itself does not exist. */ Optional estimateSubmittedQuery(UUID queryRequestId); + + /** + * The warehouse's pre-flight bytes-scanned estimate for a statement that has no persisted + * estimate of its own (#941 — a request-group member), dry-run under the same + * {@code accessflow.proxy.estimate-timeout} as the submission estimate. Nothing is persisted. + * Empty when the engine reports no bytes figure, the plan is unsupported, or the dry-run fails — + * never throws. + */ + Optional estimateBytesScanned(QueryExecutionRequest request); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java b/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java index aebd2a74a..62d4e5e97 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java @@ -84,7 +84,7 @@ public Optional estimateSubmittedQuery(UUID queryRequestI } catch (RuntimeException ex) { log.warn("Cost estimate failed for query {}: {}", queryRequestId, ex.getMessage()); var command = new PersistQueryEstimateCommand(null, snapshot.queryType(), false, null, - null, null, null, null, null, null, true, truncate(ex.getMessage()), + null, null, null, null, null, null, null, true, truncate(ex.getMessage()), durationMs(start)); var result = persistAndPublish(queryRequestId, command, false); eventPublisher.publishEvent( @@ -93,6 +93,25 @@ null, null, null, null, null, null, true, truncate(ex.getMessage()), } } + @Override + public Optional estimateBytesScanned(QueryExecutionRequest request) { + var bounded = new QueryExecutionRequest(request.datasourceId(), request.sql(), + request.queryType(), request.maxRowsOverride(), properties.estimateTimeout(), + request.restrictedColumns(), request.columnMasks(), + request.rowSecurityPredicates(), request.transactional(), request.statements(), + request.softDeleteDirectives(), request.referencedTables()); + try { + var dryRun = queryExecutor.dryRun(bounded); + return dryRun.supported() + ? Optional.ofNullable(dryRun.estimatedBytesScanned()) + : Optional.empty(); + } catch (RuntimeException ex) { + log.debug("Bytes estimate failed for datasource {}: {}", request.datasourceId(), + ex.getMessage()); + return Optional.empty(); + } + } + private QueryExecutionRequest buildRequest(QueryRequestSnapshot snapshot) { var rowSecurityPredicates = rowSecurityResolutionService .resolveApplicable(snapshot.organizationId(), snapshot.datasourceId(), @@ -144,6 +163,7 @@ private PersistQueryEstimateCommand toCommand(QueryRequestSnapshot snapshot, affectedRows, access != null ? truncateTo(access.operation(), 128) : null, root != null ? root.estimatedCost() : null, + dryRun.estimatedBytesScanned(), planJson(root, redactPredicates), redactPredicates ? null : dryRun.rawPlan(), null, false, null, durationMs); } @@ -167,14 +187,14 @@ private static QueryPlanNode accessNode(QueryPlanNode root, QueryType queryType) private PersistQueryEstimateCommand unsupportedCommand(QueryRequestSnapshot snapshot, String engineId, String reason) { return new PersistQueryEstimateCommand(engineId, snapshot.queryType(), false, null, null, - null, null, null, null, truncate(reason), false, null, null); + null, null, null, null, null, truncate(reason), false, null, null); } private static PersistQueryEstimateCommand withAffectedRows(PersistQueryEstimateCommand command, Long affectedRows, int durationMs) { return new PersistQueryEstimateCommand(command.engineId(), command.queryType(), command.supported(), command.estimatedRows(), affectedRows, command.scanType(), - command.estimatedCost(), command.planJson(), command.rawPlan(), + command.estimatedCost(), command.estimatedBytesScanned(), command.planJson(), command.rawPlan(), command.unsupportedReason(), command.failed(), command.errorMessage(), durationMs); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java index 6227e9e31..aec847e09 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java @@ -6,6 +6,9 @@ import com.bablsoft.accessflow.audit.api.AuditEntry; import com.bablsoft.accessflow.audit.api.AuditLogService; import com.bablsoft.accessflow.audit.api.AuditResourceType; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; +import com.bablsoft.accessflow.core.api.BytesScannedCapExceededException; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; import com.bablsoft.accessflow.core.api.ColumnMaskDirective; import com.bablsoft.accessflow.core.api.DatasourceLookupService; import com.bablsoft.accessflow.core.api.DatasourceUserPermissionLookupService; @@ -17,7 +20,11 @@ import com.bablsoft.accessflow.core.api.RowSecurityResolutionService; import com.bablsoft.accessflow.core.api.SelectExecutionResult; import com.bablsoft.accessflow.core.api.UpdateExecutionResult; +import com.bablsoft.accessflow.proxy.api.QueryCostEstimateService; import com.bablsoft.accessflow.proxy.api.QueryExecutor; +import com.bablsoft.accessflow.requestgroups.internal.persistence.repo.GroupReviewDecisionRepository; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.DecisionType; import com.bablsoft.accessflow.proxy.api.QueryParser; import com.bablsoft.accessflow.requestgroups.api.RequestGroupItemStatus; import com.bablsoft.accessflow.requestgroups.api.RequestGroupStatus; @@ -31,9 +38,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.MessageSource; +import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Service; import java.time.Instant; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -66,6 +76,10 @@ public class GroupExecutionService { private final ApiInlineExecutionService apiInlineExecutionService; private final AuditLogService auditLogService; private final ApplicationEventPublisher eventPublisher; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; + private final MessageSource messageSource; + private final QueryCostEstimateService queryCostEstimateService; + private final GroupReviewDecisionRepository decisionRepository; /** Execute an APPROVED group. Idempotent: silently returns if it is not APPROVED (or not yet due). */ public void execute(UUID groupId, UUID actorUserId, String trigger) { @@ -172,10 +186,12 @@ private void runQuery(RequestGroupEntity group, RequestGroupItemEntity item) { if (appliedRowLimit.isPresent()) { rowLimitOverride = appliedRowLimit.get().tighten(rowLimitOverride); } - var result = queryExecutor.execute(new QueryExecutionRequest( + var request = new QueryExecutionRequest( item.getDatasourceId(), item.getSqlText(), item.getQueryType(), rowLimitOverride, null, restrictedColumns, columnMasks, rowSecurity, parsed.transactional(), - parsed.statements(), List.of(), parsed.referencedTables())); + parsed.statements(), List.of(), parsed.referencedTables()); + enforceBytesScannedCap(group, item, request); + var result = queryExecutor.execute(request); long rows = switch (result) { case SelectExecutionResult select -> select.rowCount(); case UpdateExecutionResult update -> update.rowsAffected(); @@ -220,6 +236,59 @@ private void auditMember(RequestGroupEntity group, RequestGroupItemEntity item, "member_status", item.getStatus().name())); } + /** + * A member has no persisted estimate of its own, so when a bytes-scanned cap (#941) binds the + * submitter the member is dry-run here — only then, so an uncapped group pays nothing. A + * refusal fails the member like any execution error ({@code continue_on_error} decides the + * rest). A missing estimate under {@code REQUIRE_REVIEW} passes only when a person approved + * the group. + */ + private void enforceBytesScannedCap(RequestGroupEntity group, RequestGroupItemEntity item, + QueryExecutionRequest request) { + var cap = bytesScannedCapResolutionService + .resolve(item.getDatasourceId(), group.getSubmittedBy()) + .orElse(null); + if (cap == null) { + return; + } + var estimated = queryCostEstimateService.estimateBytesScanned(request).orElse(null); + var outcome = cap.check(estimated); + // A group can be auto-approved by its plans without anyone looking at it, and the cap is + // not consulted at that point — so "require review" is only satisfied when a person + // actually approved the group. Otherwise the member is refused rather than run unreviewed. + boolean unreviewed = outcome == BytesScannedCapOutcome.NO_ESTIMATE_REVIEW + && !decisionRepository.existsByRequestGroupIdAndDecision(group.getId(), + DecisionType.APPROVED); + if (!outcome.rejects() && !unreviewed) { + return; + } + var limit = ByteSizeFormat.format(cap.limit()); + var locale = LocaleContextHolder.getLocale(); + String message; + if (estimated != null) { + message = messageSource.getMessage("error.bytes_cap.exceeded", + new Object[]{ByteSizeFormat.format(estimated), limit}, locale); + } else if (unreviewed) { + message = messageSource.getMessage("error.bytes_cap.no_estimate_unreviewed", + new Object[]{limit}, locale); + } else { + message = messageSource.getMessage("error.bytes_cap.no_estimate", new Object[]{limit}, + locale); + } + var metadata = new HashMap(); + metadata.put("trigger", "bytes_scanned_cap"); + metadata.put("stage", "execution"); + metadata.put("item_id", item.getId().toString()); + metadata.put("limit", cap.limit()); + metadata.put("source", cap.source().name()); + if (estimated != null) { + metadata.put("estimated_bytes", estimated); + } + metadata.put("outcome", outcome.name()); + audit(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, group, null, metadata); + throw new BytesScannedCapExceededException(message, cap, estimated, outcome); + } + private void audit(AuditAction action, RequestGroupEntity group, UUID actorId, Map metadata) { try { diff --git a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java index 1c6ab3e48..2fd025c84 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java +++ b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java @@ -18,5 +18,8 @@ public interface GroupReviewDecisionRepository extends JpaRepository findByRequestGroupIdAndReviewerIdAndStage( UUID requestGroupId, UUID reviewerId, int stage); + /** Whether any person approved the group — the review a missing bytes estimate demands (#941). */ + boolean existsByRequestGroupIdAndDecision(UUID requestGroupId, DecisionType decision); + long countByRequestGroupIdAndStageAndDecision(UUID requestGroupId, int stage, DecisionType decision); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java index 87dabc4c6..8e45ae795 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java @@ -143,7 +143,7 @@ DatasourcePageResponse listDatasources(Authentication authentication, Pageable p @ApiResponse(responseCode = "201", description = "Datasource created") @ApiResponse(responseCode = "400", description = "Validation error") @ApiResponse(responseCode = "409", description = "A datasource with this name already exists") - @ApiResponse(responseCode = "422", description = "JDBC driver for the selected db_type cannot be resolved") + @ApiResponse(responseCode = "422", description = "JDBC driver for the selected db_type cannot be resolved, or a bytes-scanned cap on an engine without a bytes estimate (BYTES_SCANNED_CAP_NOT_SUPPORTED)") ResponseEntity createDatasource( @Valid @RequestBody CreateDatasourceRequest request, Authentication authentication, @@ -176,7 +176,9 @@ ResponseEntity createDatasource( request.resultCacheEnabled(), request.resultCacheTtlSeconds(), request.privateKeyPassphrase(), - request.environment()); + request.environment(), + request.maxBytesScannedPerQuery(), + request.bytesCapMissingEstimate()); var created = datasourceAdminService.create(command); recordAudit(AuditAction.DATASOURCE_CREATED, AuditResourceType.DATASOURCE, created.id(), caller, auditContext, Map.of("name", created.name(), "db_type", created.dbType().name())); @@ -206,6 +208,7 @@ DatasourceResponse getDatasource(@PathVariable UUID id, Authentication authentic @ApiResponse(responseCode = "400", description = "Validation error") @ApiResponse(responseCode = "404", description = "Datasource not found") @ApiResponse(responseCode = "409", description = "Name conflict with another datasource") + @ApiResponse(responseCode = "422", description = "Bytes-scanned cap on an engine without a bytes estimate (BYTES_SCANNED_CAP_NOT_SUPPORTED)") DatasourceResponse updateDatasource(@PathVariable UUID id, @Valid @RequestBody UpdateDatasourceRequest request, Authentication authentication, @@ -237,7 +240,10 @@ DatasourceResponse updateDatasource(@PathVariable UUID id, request.resultCacheTtlSeconds(), request.privateKeyPassphrase(), request.environment(), - request.clearEnvironment()); + request.clearEnvironment(), + request.maxBytesScannedPerQuery(), + request.clearMaxBytesScannedPerQuery(), + request.bytesCapMissingEstimate()); var updated = datasourceAdminService.update(id, caller.organizationId(), command); recordAudit(AuditAction.DATASOURCE_UPDATED, AuditResourceType.DATASOURCE, id, caller, auditContext, Map.of("name", updated.name())); @@ -335,7 +341,8 @@ PermissionListResponse listPermissions(@PathVariable UUID id, Authentication aut @ApiResponse(responseCode = "409", description = "Permission already exists for this user") @ApiResponse(responseCode = "422", description = "Target user is not in the organization, or denied_columns / " - + "denied_shapes is not supported by the datasource engine") + + "denied_shapes / bytes_scanned_limit_override is not supported by the " + + "datasource engine") ResponseEntity grantPermission( @PathVariable UUID id, @Valid @RequestBody CreatePermissionRequest request, @@ -349,6 +356,7 @@ ResponseEntity grantPermission( request.canDdl(), request.canBreakGlass(), request.rowLimitOverride(), + request.bytesScannedLimitOverride(), request.allowedSchemas(), request.allowedTables(), request.restrictedColumns(), @@ -425,8 +433,8 @@ GroupPermissionListResponse listGroupPermissions(@PathVariable UUID id, @ApiResponse(responseCode = "404", description = "Datasource or group not found") @ApiResponse(responseCode = "409", description = "Permission already exists for this group") @ApiResponse(responseCode = "422", - description = "denied_columns or denied_shapes is not supported by the datasource " - + "engine") + description = "denied_columns, denied_shapes or bytes_scanned_limit_override is not " + + "supported by the datasource engine") ResponseEntity grantGroupPermission( @PathVariable UUID id, @Valid @RequestBody CreateGroupPermissionRequest request, @@ -440,6 +448,7 @@ ResponseEntity grantGroupPermission( request.canDdl(), request.canBreakGlass(), request.rowLimitOverride(), + request.bytesScannedLimitOverride(), request.allowedSchemas(), request.allowedTables(), request.restrictedColumns(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java index 7dfbd79ba..668b5ca86 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java @@ -20,6 +20,7 @@ import com.bablsoft.accessflow.core.api.EmailAlreadyExistsException; import com.bablsoft.accessflow.core.api.DeniedColumnsNotSupportedException; import com.bablsoft.accessflow.core.api.DeniedShapesNotSupportedException; +import com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException; import com.bablsoft.accessflow.core.api.IllegalDatasourcePermissionException; import com.bablsoft.accessflow.core.api.DataClassificationTagNotFoundException; import com.bablsoft.accessflow.core.api.IllegalDataClassificationTagException; @@ -433,6 +434,16 @@ ProblemDetail handleDeniedShapesNotSupported(DeniedShapesNotSupportedException e return pd; } + @ExceptionHandler(BytesScannedCapNotSupportedException.class) + ProblemDetail handleBytesScannedCapNotSupported(BytesScannedCapNotSupportedException ex) { + var pd = ProblemDetail.forStatusAndDetail(HttpStatus.UNPROCESSABLE_CONTENT, + msg("error.bytes_scanned_cap_not_supported", ex.dbType().name())); + pd.setProperty("error", "BYTES_SCANNED_CAP_NOT_SUPPORTED"); + pd.setProperty("dbType", ex.dbType().name()); + pd.setProperty("timestamp", Instant.now().toString()); + return pd; + } + @ExceptionHandler(MaskingPolicyNotFoundException.class) ProblemDetail handleMaskingPolicyNotFound(MaskingPolicyNotFoundException ex) { var pd = ProblemDetail.forStatusAndDetail(HttpStatus.NOT_FOUND, diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java index 3252e31d7..da157ef06 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.security.internal.web.model; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.SslMode; @@ -59,5 +60,8 @@ public record CreateDatasourceRequest( @Size(max = 1024, message = "{validation.private_key_passphrase.max}") String privateKeyPassphrase, // #861: optional; an unset environment resolves to the org-wide default SQL review ruleset. - DatasourceEnvironment environment + DatasourceEnvironment environment, + // #941: optional bytes-scanned cap; only accepted on BigQuery / Snowflake / Databricks. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) {} diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java index 1db6b7b7c..2ed41091a 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java @@ -19,6 +19,8 @@ public record CreateGroupPermissionRequest( Boolean canDdl, Boolean canBreakGlass, @Min(value = 1, message = "{validation.row_limit.min}") Integer rowLimitOverride, + // #941: bytes-scanned cap for this grant; only accepted on bytes-reporting engines. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List<@NotBlank(message = "{validation.restricted_columns.item_blank}") String> restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java index a5fa7e22f..66f66cd0e 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java @@ -19,6 +19,8 @@ public record CreatePermissionRequest( Boolean canDdl, Boolean canBreakGlass, @Min(value = 1, message = "{validation.row_limit.min}") Integer rowLimitOverride, + // #941: bytes-scanned cap for this grant; only accepted on bytes-reporting engines. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List<@NotBlank(message = "{validation.restricted_columns.item_blank}") String> restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java index d75db4538..27e409269 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.security.internal.web.model; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.DatasourceView; import com.bablsoft.accessflow.core.api.DbType; @@ -36,7 +37,9 @@ public record DatasourceResponse( String localDatacenter, boolean resultCacheEnabled, Integer resultCacheTtlSeconds, - DatasourceEnvironment environment + DatasourceEnvironment environment, + Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { /** One read-replica endpoint — never carries the password. */ public record ReadReplicaResponse(UUID id, String jdbcUrl, String username) { @@ -73,6 +76,8 @@ public static DatasourceResponse from(DatasourceView view) { view.localDatacenter(), view.resultCacheEnabled(), view.resultCacheTtlSeconds(), - view.environment()); + view.environment(), + view.maxBytesScannedPerQuery(), + view.bytesCapMissingEstimate()); } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java index 75f6afd43..21e75d73b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java @@ -18,6 +18,7 @@ public record GroupPermissionResponse( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, @@ -41,6 +42,7 @@ public static GroupPermissionResponse from(DatasourceGroupPermissionView view) { view.canDdl(), view.canBreakGlass(), view.rowLimitOverride(), + view.bytesScannedLimitOverride(), view.allowedSchemas(), view.allowedTables(), view.restrictedColumns(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java index 4b4e2a939..430b82132 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java @@ -18,6 +18,7 @@ public record PermissionResponse( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, @@ -41,6 +42,7 @@ public static PermissionResponse from(DatasourcePermissionView view) { view.canDdl(), view.canBreakGlass(), view.rowLimitOverride(), + view.bytesScannedLimitOverride(), view.allowedSchemas(), view.allowedTables(), view.restrictedColumns(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java index 00a4b2b07..5cfe9a335 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.security.internal.web.model; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.SslMode; import jakarta.validation.Valid; @@ -52,5 +53,9 @@ public record UpdateDatasourceRequest( String privateKeyPassphrase, // #861: null leaves the environment unchanged; clearEnvironment=true unsets it. DatasourceEnvironment environment, - Boolean clearEnvironment + Boolean clearEnvironment, + // #941: null leaves the cap unchanged; clearMaxBytesScannedPerQuery=true removes it. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long maxBytesScannedPerQuery, + Boolean clearMaxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) {} diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java index 1dea44e52..a7f31d68a 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java @@ -36,6 +36,10 @@ * {@code scanType} is the plan's root operation (e.g. {@code Seq Scan}). Both are {@code null} * when the estimate is absent, unsupported, or failed — the matching conditions fail closed. * + *

{@code estimatedBytesScanned} (#941) is the warehouse engines' pre-flight scan estimate in raw + * bytes from the same row — {@code null} for every engine that reports none, and whenever the + * estimate is absent or failed. + * *

{@code queryShapes} are the structural features re-derived from the SQL with the WHERE / LIMIT * signals (#940). {@code shapesAnalyzed} is {@code false} when the SQL could not be parsed or walked * (typically a non-SQL engine); the {@code query_shape} condition then fails closed. @@ -59,7 +63,8 @@ public record ConditionContext( Long estimatedRows, String scanType, Set queryShapes, - boolean shapesAnalyzed) { + boolean shapesAnalyzed, + Long estimatedBytesScanned) { public ConditionContext { referencedTables = Set.copyOf(referencedTables == null ? Set.of() : referencedTables); @@ -67,6 +72,21 @@ public record ConditionContext( queryShapes = Set.copyOf(queryShapes == null ? Set.of() : queryShapes); } + /** Backward-compatible constructor without the #941 bytes estimate (defaults to absent). */ + public ConditionContext(QueryType queryType, Set referencedTables, RiskLevel riskLevel, + int riskScore, String requesterRoleName, Set requesterGroupIds, + LocalDateTime evaluatedAt, boolean hasWhereClause, + boolean hasLimitClause, boolean transactional, + String requesterIpAddress, String requesterUserAgent, + boolean ciCdOrigin, Integer minutesSinceLastApproval, + boolean anomalyActive, Long estimatedRows, String scanType, + Set queryShapes, boolean shapesAnalyzed) { + this(queryType, referencedTables, riskLevel, riskScore, requesterRoleName, + requesterGroupIds, evaluatedAt, hasWhereClause, hasLimitClause, transactional, + requesterIpAddress, requesterUserAgent, ciCdOrigin, minutesSinceLastApproval, + anomalyActive, estimatedRows, scanType, queryShapes, shapesAnalyzed, null); + } + /** Backward-compatible constructor without the #940 shape signals (defaults to not analyzed). */ public ConditionContext(QueryType queryType, Set referencedTables, RiskLevel riskLevel, int riskScore, String requesterRoleName, Set requesterGroupIds, @@ -92,7 +112,7 @@ public ConditionContext(QueryType queryType, Set referencedTables, RiskL this(queryType, referencedTables, riskLevel, riskScore, requesterRoleName, requesterGroupIds, evaluatedAt, hasWhereClause, hasLimitClause, transactional, requesterIpAddress, requesterUserAgent, ciCdOrigin, minutesSinceLastApproval, - anomalyActive, null, null, Set.of(), false); + anomalyActive, null, null, Set.of(), false, null); } /** @return {@code true} when an AI risk level / score signal is present. */ diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java index a273ec809..1a9e3d3d2 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java @@ -215,6 +215,25 @@ record EstimatedRows(ComparisonOperator operator, long value) implements Conditi } } + /** + * Compares the warehouse engines' pre-flight bytes-scanned estimate (#941 — BigQuery dry-run + * {@code totalBytesProcessed}, Snowflake {@code bytesAssigned}, Databricks {@code sizeInBytes}) + * with {@code value}, in raw bytes. Fails closed: evaluates to {@code false} + * when no bytes estimate exists — every engine without one, a failed or unsupported estimate — + * so it is an escalation trigger, never a way to auto-approve on missing context. + */ + record EstimatedBytesScanned(ComparisonOperator operator, long value) implements ConditionNode { + public EstimatedBytesScanned { + if (operator == null) { + throw new IllegalArgumentException( + "EstimatedBytesScanned condition requires an operator"); + } + if (value < 0) { + throw new IllegalArgumentException("EstimatedBytesScanned value must be >= 0"); + } + } + } + /** * Matches when the pre-flight plan's root scan/operation type (AF-624 — e.g. {@code Seq Scan}, * {@code Index Scan}, {@code COLLSCAN}) matches any glob in {@code patterns} ({@code *} = any diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java index 71d336b9f..2b76abb70 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java @@ -12,7 +12,8 @@ * missing stage is always a bug rather than a normal outcome. * *

Only {@link #ROUTING_POLICIES}, {@link #GRANT_FAST_PATH} and {@link #REVIEW_PLAN} are decided - * on the live asynchronous path; {@link #SQL_REVIEW} is evaluated synchronously at submission and + * on the live asynchronous path, with {@link #BYTES_SCANNED_CAP} decided just before them; + * {@link #SQL_REVIEW} is evaluated synchronously at submission and * read back there. The rest happen elsewhere in production — the first four in the * synchronous submission gate, {@link #ROW_SECURITY} and {@link #MASKING} at execution time, * {@link #ELIGIBLE_REVIEWERS} in notification fan-out, and {@link #BREAK_GLASS} in a separate @@ -40,6 +41,14 @@ public enum QueryDecisionStepKind implements DecisionStepKind { */ SQL_REVIEW, + /** + * The bytes-scanned cap (#941): {@code DENY} when the pre-flight estimate exceeds the cap, or + * no estimate exists and the datasource rejects on a missing one; {@code MATCH} when a missing + * estimate forces human review (every auto-approve stage below is then suppressed); + * {@code ALLOW} when the estimate is within the cap; {@code NO_MATCH} when no cap applies. + */ + BYTES_SCANNED_CAP, + /** Every enabled routing policy in ascending priority order, matched and unmatched. */ ROUTING_POLICIES, diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java new file mode 100644 index 000000000..731f35355 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java @@ -0,0 +1,42 @@ +package com.bablsoft.accessflow.workflow.internal; + +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; + +import java.util.Objects; + +/** + * The bytes-scanned cap (#941) that applies to one request, and how its estimate compared — the + * input {@link QueryDecisionEvaluator} decides on, like the {@code BLOCK} rule ids. + * + * @param estimatedBytes the persisted pre-flight estimate, {@code null} when there is none + * @param outcome the comparison, or {@code null} when it cannot be made at all: the access + * simulator has no submitted query and therefore no estimate, so it reports + * the cap without letting a guessed outcome decide + */ +record BytesCapCheck(long limit, BytesScannedCapSource source, Long estimatedBytes, + BytesScannedCapOutcome outcome) { + + BytesCapCheck { + Objects.requireNonNull(source, "source"); + } + + static BytesCapCheck of(AppliedBytesCap cap, Long estimatedBytes) { + return new BytesCapCheck(cap.limit(), cap.source(), estimatedBytes, + cap.check(estimatedBytes)); + } + + /** A cap the caller knows applies but cannot compare (the simulator). */ + static BytesCapCheck unevaluated(AppliedBytesCap cap) { + return new BytesCapCheck(cap.limit(), cap.source(), null, null); + } + + boolean rejects() { + return outcome != null && outcome.rejects(); + } + + boolean forcesReview() { + return outcome == BytesScannedCapOutcome.NO_ESTIMATE_REVIEW; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java index f8dec205d..532972844 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.workflow.internal; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; import com.bablsoft.accessflow.core.api.ApproverRule; import com.bablsoft.accessflow.core.api.ColumnRefKeys; import com.bablsoft.accessflow.core.api.DatasourceAdminService; @@ -87,6 +88,7 @@ class DefaultAccessSimulationService implements AccessSimulationService { private final RowSecurityClassificationService rowSecurityClassificationService; private final MaskingPolicyResolutionService maskingPolicyResolutionService; private final BreakGlassEligibilityService breakGlassEligibilityService; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; // Time-of-day / day-of-week routing conditions evaluate in the server's local zone, so the // simulator has to use the same zone the live listener does. Deliberately NOT the injected @@ -140,9 +142,16 @@ public AccessSimulationResult simulate(UUID organizationId, AccessSimulationInpu .distinct() .sorted() .toList(); + // A hypothetical request has no pre-flight estimate (COST_ESTIMATE_ABSENT), so a bytes cap + // is reported but not compared: guessing "no estimate" would show a refusal or forced + // review that a real submission with an estimate would not get (#941). + var bytesCap = bytesScannedCapResolutionService + .resolve(input.datasourceId(), input.userId()) + .map(BytesCapCheck::unevaluated) + .orElse(null); var decision = queryDecisionEvaluator.evaluate( syntheticSnapshot(organizationId, input, parsed), input.aiOutcome(), - input.riskLevel(), input.effectiveRiskScore(), blockingRuleIds, clock); + input.riskLevel(), input.effectiveRiskScore(), blockingRuleIds, bytesCap, clock); steps.addAll(withFullPolicyList(decision, organizationId, input.datasourceId())); steps.add(reviewerStep(input, decision.nextStatus())); diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java index c47171bc5..1449a6bba 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java @@ -1,5 +1,10 @@ package com.bablsoft.accessflow.workflow.internal; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; +import com.bablsoft.accessflow.core.api.BytesScannedCapExceededException; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; +import com.bablsoft.accessflow.core.api.QueryEstimateSnapshot; +import com.bablsoft.accessflow.proxy.api.QueryCostEstimateService; import com.bablsoft.accessflow.audit.api.AuditAction; import com.bablsoft.accessflow.audit.api.AuditEntry; import com.bablsoft.accessflow.audit.api.AuditLogService; @@ -87,6 +92,8 @@ class DefaultQueryLifecycleService implements QueryLifecycleService { private final ObjectMapper objectMapper; private final MessageSource messageSource; private final ApplicationEventPublisher eventPublisher; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; + private final QueryCostEstimateService queryCostEstimateService; private String msg(String key) { return messageSource.getMessage(key, null, LocaleContextHolder.getLocale()); @@ -277,6 +284,10 @@ private ExecutionOutcome doExecute(QueryRequestSnapshot query, UUID actorUserId, AuditAction successAction) { var startedAt = Instant.now(); try { + // #941: re-checked here, not only when the query left PENDING_AI — scheduled, recurring + // and break-glass runs execute later or without that decision, and the cap may have + // been lowered since. A refusal is recorded as a failed execution. + var bytesCap = enforceBytesScannedCap(query); var permission = permissionLookupService .findFor(query.submittedByUserId(), query.datasourceId()); var restrictedColumns = permission @@ -398,6 +409,13 @@ private ExecutionOutcome doExecute(QueryRequestSnapshot query, UUID actorUserId, appliedRowSecurityPolicyIds.stream() .map(UUID::toString).sorted().toList()); } + if (bytesCap != null) { + successMetadata.put("bytes_scanned_cap", bytesCap.limit()); + successMetadata.put("bytes_scanned_cap_source", bytesCap.source().name()); + if (bytesCap.estimatedBytes() != null) { + successMetadata.put("bytes_scanned_estimate", bytesCap.estimatedBytes()); + } + } if (!appliedRowLimitPolicyIds.isEmpty()) { successMetadata.put("applied_row_limit_policy_ids", appliedRowLimitPolicyIds.stream() @@ -423,6 +441,56 @@ private ExecutionOutcome doExecute(QueryRequestSnapshot query, UUID actorUserId, } } + /** + * @return the cap that applied and the estimate it was compared with, or {@code null} when no + * cap binds the submitter here + * @throws BytesScannedCapExceededException when the estimate exceeds the cap, or is missing and + * the datasource rejects on a missing estimate. A missing estimate under + * {@code REQUIRE_REVIEW} passes: the review it demands happened before approval. + */ + private BytesCapCheck enforceBytesScannedCap(QueryRequestSnapshot query) { + var cap = bytesScannedCapResolutionService + .resolve(query.datasourceId(), query.submittedByUserId()) + .orElse(null); + if (cap == null) { + return null; + } + Long estimated; + try { + estimated = queryCostEstimateService.estimateSubmittedQuery(query.id()) + .filter(e -> !e.failed()) + .map(QueryEstimateSnapshot::estimatedBytesScanned) + .orElse(null); + } catch (RuntimeException ex) { + log.warn("Pre-flight estimate unavailable for query {} at execution: {}", query.id(), + ex.getMessage()); + estimated = null; + } + var check = BytesCapCheck.of(cap, estimated); + if (!check.rejects()) { + return check; + } + var limit = ByteSizeFormat.format(cap.limit()); + var message = estimated == null + ? messageSource.getMessage("error.bytes_cap.no_estimate", new Object[]{limit}, + LocaleContextHolder.getLocale()) + : messageSource.getMessage("error.bytes_cap.exceeded", + new Object[]{ByteSizeFormat.format(estimated), limit}, + LocaleContextHolder.getLocale()); + var metadata = new HashMap(); + metadata.put("trigger", "bytes_scanned_cap"); + metadata.put("stage", "execution"); + metadata.put("limit", cap.limit()); + metadata.put("source", cap.source().name()); + if (estimated != null) { + metadata.put("estimated_bytes", estimated); + } + metadata.put("outcome", check.outcome().name()); + recordAudit(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, query.id(), null, + query.organizationId(), metadata); + throw new BytesScannedCapExceededException(message, cap, estimated, check.outcome()); + } + private ExecutionOutcome recordFailure(QueryRequestSnapshot query, UUID actorUserId, String trigger, Instant startedAt, RuntimeException ex) { var completedAt = Instant.now(); diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java index 483f1e0bd..afbd0d197 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java @@ -25,17 +25,30 @@ * into human review (#864), or {@code null} when the finding changed * nothing — no block, {@code WARN} only, or the request was headed to * review regardless + * @param bytesCap the bytes-scanned cap that applied (#941), or {@code null} when none + * @param bytesCapChangedOutcome whether the cap refused the query or turned an automatic approval + * into human review — the condition for its audit row */ record QueryDecision(QueryDecisionKind kind, QueryStatus nextStatus, RoutingMatch routingMatch, Integer effectiveApprovals, UUID grantId, String grantApproverEmail, ConditionContext context, DecisionTrace trace, - SqlReviewSuppression sqlReviewSuppression) { + SqlReviewSuppression sqlReviewSuppression, BytesCapCheck bytesCap, + boolean bytesCapChangedOutcome) { + + /** A decision no bytes-scanned cap took part in. */ + QueryDecision(QueryDecisionKind kind, QueryStatus nextStatus, RoutingMatch routingMatch, + Integer effectiveApprovals, UUID grantId, String grantApproverEmail, + ConditionContext context, DecisionTrace trace, + SqlReviewSuppression sqlReviewSuppression) { + this(kind, nextStatus, routingMatch, effectiveApprovals, grantId, grantApproverEmail, context, + trace, sqlReviewSuppression, null, false); + } /** A decision no SQL review finding interfered with. */ QueryDecision(QueryDecisionKind kind, QueryStatus nextStatus, RoutingMatch routingMatch, Integer effectiveApprovals, UUID grantId, String grantApproverEmail, ConditionContext context, DecisionTrace trace) { this(kind, nextStatus, routingMatch, effectiveApprovals, grantId, grantApproverEmail, context, - trace, null); + trace, (SqlReviewSuppression) null); } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java index 357b66216..bdfd66c54 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java @@ -2,6 +2,7 @@ import com.bablsoft.accessflow.access.api.AccessGrantLookupService; import com.bablsoft.accessflow.access.api.AccessGrantView; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; import com.bablsoft.accessflow.core.api.QueryRequestSnapshot; import com.bablsoft.accessflow.core.api.QueryStatus; import com.bablsoft.accessflow.core.api.QueryType; @@ -68,17 +69,31 @@ class QueryDecisionEvaluator { private final RoutingPolicyEngine routingPolicyEngine; private final AccessGrantLookupService accessGrantLookupService; + /** {@link #evaluate(QueryRequestSnapshot, AiOutcome, RiskLevel, int, List, BytesCapCheck, Clock)} with no bytes cap. */ + QueryDecision evaluate(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLevel riskLevel, + int riskScore, List blockingRuleIds, Clock clock) { + return evaluate(query, aiOutcome, riskLevel, riskScore, blockingRuleIds, null, clock); + } + /** * @param riskScore the AI's numeric score, or {@code -1} when there is none — the same * "absent" sentinel the live completion event uses * @param blockingRuleIds the distinct SQL review rule ids that fired at {@code BLOCK} for this * request, empty when none did (#864) + * @param bytesCap the bytes-scanned cap that applies (#941), {@code null} when none does */ QueryDecision evaluate(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLevel riskLevel, - int riskScore, List blockingRuleIds, Clock clock) { + int riskScore, List blockingRuleIds, BytesCapCheck bytesCap, + Clock clock) { var block = blockingRuleIds == null ? List.of() : List.copyOf(blockingRuleIds); + var guard = new Guard(block, bytesCap); + // A hard cap is a refusal, not a routing signal: it decides before routing and before the + // AI-failure fallback, so no policy or plan can approve a query the cap has refused. + if (bytesCap != null && bytesCap.rejects()) { + return bytesCapRejected(block, bytesCap); + } if (aiOutcome == AiOutcome.FAILED) { - return aiFailed(block); + return aiFailed(block, bytesCap); } // Only a COMPLETED analysis carries a risk signal. Normalising here rather than trusting the // caller keeps the SKIPPED branch identical to production, where the listener passes no risk @@ -88,28 +103,100 @@ QueryDecision evaluate(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLeve var plan = reviewPlanLookupService.findForDatasource(query.datasourceId()).orElse(null); var context = conditionContextFactory.forLiveQuery(query, effectiveRisk, effectiveScore, clock); - var steps = new ArrayList(4); + var steps = new ArrayList(5); steps.add(sqlReviewStep(block)); + steps.add(bytesCapStep(bytesCap)); var match = routingPolicyEngine.evaluate(query.organizationId(), query.datasourceId(), context).orElse(null); if (match != null) { - return routed(match, plan, context, steps, block); + return routed(match, plan, context, steps, guard); } steps.add(DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.NO_MATCH, "workflow.decision.routing.no_match")); var suppressed = new ArrayList(2); - var grant = findCoveringGrant(query, context, steps, block, suppressed); + var grant = findCoveringGrant(query, context, steps, guard, suppressed); if (grant != null) { steps.add(DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.SKIP, "workflow.decision.plan.skipped_grant_covered", planDetails(plan))); return new QueryDecision(QueryDecisionKind.GRANT_FAST_PATH, QueryStatus.APPROVED, null, null, grant.id(), grant.approverEmail(), context, - new DecisionTrace(steps, QueryStatus.APPROVED)); + new DecisionTrace(steps, QueryStatus.APPROVED), null, bytesCap, false); + } + + return planned(query, plan, effectiveRisk, context, steps, guard, suppressed); + } + + /** + * What may turn an automatic approval into human review: a {@code BLOCK} SQL review finding + * (#864) and a bytes-scanned cap with no estimate to compare against (#941). The trace names + * the SQL review first when both apply. + */ + private record Guard(List block, BytesCapCheck bytesCap) { + + boolean suppresses() { + return !block.isEmpty() || (bytesCap != null && bytesCap.forcesReview()); + } + + boolean capForcesReview() { + return bytesCap != null && bytesCap.forcesReview(); } - return planned(query, plan, effectiveRisk, context, steps, block, suppressed); + String reasonSuffix() { + return block.isEmpty() ? "bytes_cap" : "sql_review"; + } + } + + /** The cap step is always present so the trace keeps one entry per stage. */ + private static DecisionTraceStep bytesCapStep(BytesCapCheck cap) { + if (cap == null) { + return DecisionTraceStep.of(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.NO_MATCH, "workflow.decision.bytes_cap.none"); + } + var details = new LinkedHashMap(); + details.put("bytes_scanned_cap", cap.limit()); + details.put("bytes_scanned_cap_source", cap.source().name()); + details.put("estimated_bytes_scanned", cap.estimatedBytes()); + details.put("bytes_scanned_cap_outcome", cap.outcome() == null ? null : cap.outcome().name()); + var limit = ByteSizeFormat.format(cap.limit()); + if (cap.outcome() == null) { + return new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.SKIP, + "workflow.decision.bytes_cap.unevaluated", List.of(limit), details); + } + return switch (cap.outcome()) { + case WITHIN -> new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.ALLOW, "workflow.decision.bytes_cap.within", + List.of(ByteSizeFormat.format(cap.estimatedBytes()), limit), details); + case EXCEEDED -> new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.DENY, "workflow.decision.bytes_cap.exceeded", + List.of(ByteSizeFormat.format(cap.estimatedBytes()), limit), details); + case NO_ESTIMATE_REVIEW -> new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.MATCH, "workflow.decision.bytes_cap.no_estimate_review", + List.of(limit), details); + case NO_ESTIMATE_REJECTED -> new DecisionTraceStep( + QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.DENY, + "workflow.decision.bytes_cap.no_estimate_rejected", List.of(limit), details); + }; + } + + /** + * The cap refused the query (#941). Nothing downstream runs — there is no routing decision to + * record and no plan to consult — which is why the context is not even built. + */ + private static QueryDecision bytesCapRejected(List block, BytesCapCheck cap) { + var steps = List.of( + sqlReviewStep(block), + bytesCapStep(cap), + DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.SKIP, + "workflow.decision.routing.skipped_bytes_cap"), + DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.SKIP, + "workflow.decision.grant.skipped_bytes_cap"), + DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.SKIP, + "workflow.decision.plan.skipped_bytes_cap")); + return new QueryDecision(QueryDecisionKind.BYTES_CAP_REJECTED, QueryStatus.REJECTED, null, + null, null, null, null, new DecisionTrace(steps, QueryStatus.REJECTED), null, cap, + true); } /** @@ -131,7 +218,7 @@ private static DecisionTraceStep sqlReviewStep(List block) { private static SqlReviewSuppression suppression(List block, List paths) { - return paths.isEmpty() ? null : new SqlReviewSuppression(block, paths); + return paths.isEmpty() || block.isEmpty() ? null : new SqlReviewSuppression(block, paths); } /** @@ -140,9 +227,10 @@ private static SqlReviewSuppression suppression(List block, * auto-decision signal — and neither does the grant fast path or the review plan. Decided before * any lookup, mirroring the live listener, which builds no context at all. */ - private static QueryDecision aiFailed(List block) { + private static QueryDecision aiFailed(List block, BytesCapCheck bytesCap) { var steps = List.of( sqlReviewStep(block), + bytesCapStep(bytesCap), DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.SKIP, "workflow.decision.routing.skipped_ai_failed"), DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.SKIP, @@ -151,7 +239,7 @@ private static QueryDecision aiFailed(List block) { "workflow.decision.plan.skipped_ai_failed")); return new QueryDecision(QueryDecisionKind.AI_FAILED_PENDING_REVIEW, QueryStatus.PENDING_REVIEW, null, null, null, null, null, - new DecisionTrace(steps, QueryStatus.PENDING_REVIEW)); + new DecisionTrace(steps, QueryStatus.PENDING_REVIEW), null, bytesCap, false); } /** @@ -161,8 +249,10 @@ private static QueryDecision aiFailed(List block) { */ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, ConditionContext context, List steps, - List block) { - boolean suppressedApprove = match.action() == RoutingAction.AUTO_APPROVE && !block.isEmpty(); + Guard guard) { + var block = guard.block(); + boolean suppressedApprove = match.action() == RoutingAction.AUTO_APPROVE + && guard.suppresses(); var effect = switch (match.action()) { case AUTO_APPROVE -> suppressedApprove ? new RoutedEffect(QueryDecisionKind.ROUTING_AUTO_APPROVE_SUPPRESSED, @@ -184,10 +274,13 @@ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, details.put("matched_policy_name", match.policyName()); details.put("action", match.action().name()); details.put("effective_min_approvals", effective); - details.put("sql_review_suppressed", suppressedApprove); + details.put("sql_review_suppressed", suppressedApprove && !block.isEmpty()); + details.put("bytes_cap_suppressed", suppressedApprove && guard.capForcesReview()); steps.add(suppressedApprove ? new DecisionTraceStep(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.MATCH, - "workflow.decision.routing.matched_auto_approve_suppressed", + block.isEmpty() + ? "workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap" + : "workflow.decision.routing.matched_auto_approve_suppressed", List.of(String.valueOf(match.policyName())), details) : new DecisionTraceStep(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.MATCH, "workflow.decision.routing.matched", @@ -198,9 +291,10 @@ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, "workflow.decision.plan.skipped_routing_decided", planDetails(plan))); return new QueryDecision(kind, nextStatus, match, effective, null, null, context, new DecisionTrace(steps, nextStatus), - suppressedApprove + suppressedApprove && !block.isEmpty() ? new SqlReviewSuppression(block, List.of(SuppressedAutoApproval.ROUTING_AUTO_APPROVE)) - : null); + : null, + guard.bytesCap(), suppressedApprove && guard.capForcesReview()); } /** @@ -217,7 +311,7 @@ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, * @return the first covering grant, or {@code null} to fall through to the review plan */ private AccessGrantView findCoveringGrant(QueryRequestSnapshot query, ConditionContext context, - List steps, List block, + List steps, Guard guard, List suppressed) { if (context.anomalyActive()) { steps.add(DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.NO_MATCH, @@ -256,10 +350,11 @@ private AccessGrantView findCoveringGrant(QueryRequestSnapshot query, ConditionC var details = consideredGrants(grants); details.put("grant_id", grant.id()); details.put("approver_email", grant.approverEmail()); - if (!block.isEmpty()) { + if (guard.suppresses()) { suppressed.add(SuppressedAutoApproval.GRANT_FAST_PATH); steps.add(new DecisionTraceStep(QueryDecisionStepKind.GRANT_FAST_PATH, - StepOutcome.NO_MATCH, "workflow.decision.grant.suppressed_sql_review", + StepOutcome.NO_MATCH, + "workflow.decision.grant.suppressed_" + guard.reasonSuffix(), List.of(String.valueOf(grant.id())), details)); return null; } @@ -281,8 +376,9 @@ private record RoutedEffect(QueryDecisionKind kind, QueryStatus nextStatus, private QueryDecision planned(QueryRequestSnapshot query, ReviewPlanSnapshot plan, RiskLevel riskLevel, ConditionContext context, - List steps, List block, + List steps, Guard guard, List suppressed) { + var block = guard.block(); var details = planDetails(plan); QueryStatus nextStatus; String reasonKey; @@ -302,12 +398,14 @@ private QueryDecision planned(QueryRequestSnapshot query, ReviewPlanSnapshot pla } // Decided un-guarded first so the trace says which plan rule WOULD have approved; the block // then overrides the status alone (#864). - if (nextStatus == QueryStatus.APPROVED && !block.isEmpty()) { + if (nextStatus == QueryStatus.APPROVED && guard.suppresses()) { suppressed.add(SuppressedAutoApproval.REVIEW_PLAN); nextStatus = QueryStatus.PENDING_REVIEW; - reasonKey = "workflow.decision.plan.suppressed_sql_review"; + reasonKey = "workflow.decision.plan.suppressed_" + guard.reasonSuffix(); } - details.put("sql_review_suppressed", suppressed.contains(SuppressedAutoApproval.REVIEW_PLAN)); + boolean planSuppressed = suppressed.contains(SuppressedAutoApproval.REVIEW_PLAN); + details.put("sql_review_suppressed", planSuppressed && !block.isEmpty()); + details.put("bytes_cap_suppressed", planSuppressed && guard.capForcesReview()); steps.add(DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, nextStatus == QueryStatus.APPROVED ? StepOutcome.ALLOW : StepOutcome.DENY, reasonKey, details)); @@ -315,7 +413,8 @@ private QueryDecision planned(QueryRequestSnapshot query, ReviewPlanSnapshot pla ? QueryDecisionKind.PLAN_APPROVED : QueryDecisionKind.PLAN_PENDING_REVIEW; return new QueryDecision(kind, nextStatus, null, null, null, null, context, - new DecisionTrace(steps, nextStatus), suppression(block, suppressed)); + new DecisionTrace(steps, nextStatus), suppression(block, suppressed), + guard.bytesCap(), guard.capForcesReview() && !suppressed.isEmpty()); } private static LinkedHashMap consideredGrants(List grants) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java index 8d56b11bb..e79fd50d1 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java @@ -30,5 +30,11 @@ enum QueryDecisionKind { PLAN_PENDING_REVIEW, /** AI analysis failed; the query goes to human review unconditionally. */ - AI_FAILED_PENDING_REVIEW + AI_FAILED_PENDING_REVIEW, + + /** + * The bytes-scanned cap (#941) refused the query: its estimate exceeds the cap, or it has none + * and the datasource rejects on a missing estimate. Decided before routing and AI failure. + */ + BYTES_CAP_REJECTED } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java index f908bbdf8..76d68ce68 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java @@ -4,7 +4,14 @@ import com.bablsoft.accessflow.audit.api.AuditEntry; import com.bablsoft.accessflow.audit.api.AuditLogService; import com.bablsoft.accessflow.audit.api.AuditResourceType; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.QueryEstimateLookupService; +import com.bablsoft.accessflow.core.api.QueryEstimateSnapshot; import com.bablsoft.accessflow.core.api.QueryRequestLookupService; +import com.bablsoft.accessflow.core.api.RiskLevel; +import com.bablsoft.accessflow.proxy.api.QueryCostEstimateService; import com.bablsoft.accessflow.core.api.QueryRequestSnapshot; import com.bablsoft.accessflow.core.api.QueryRequestStateService; import com.bablsoft.accessflow.core.api.QueryStatus; @@ -25,6 +32,9 @@ import org.springframework.context.MessageSource; import org.springframework.modulith.events.ApplicationModuleListener; import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.time.Clock; import java.util.LinkedHashMap; @@ -53,6 +63,11 @@ * when any are present. When that actually changed the outcome, one {@code SQL_REVIEW_BLOCKED} audit * row is written here — system-attributed, {@code trigger=sql_review}. * + *

The bytes-scanned cap (#941) is resolved and compared against the persisted pre-flight estimate + * at every entry point and handed to the evaluator; the cap, its source and the comparison are + * stamped on the query before the decision is applied, and a {@code QUERY_BYTES_SCANNED_CAP_ENFORCED} + * audit row is written when the cap refused the query or forced it to review. + * *

AI failure unconditionally lands in {@code PENDING_REVIEW} so a human can inspect the query. The * skipped path (datasource has {@code ai_analysis_enabled = false}) runs routing with no risk signal * — risk-based conditions evaluate to {@code false} — and otherwise respects @@ -73,6 +88,10 @@ class QueryReviewStateMachine { private final AuditLogService auditLogService; private final MessageSource messageSource; private final ApplicationEventPublisher eventPublisher; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; + private final QueryCostEstimateService queryCostEstimateService; + private final QueryEstimateLookupService queryEstimateLookupService; + private final PlatformTransactionManager transactionManager; // Time-of-day / day-of-week routing conditions evaluate in the server's local zone. A field // (not an injected bean) so it can be overridden in tests without colliding with the proxy's @@ -85,29 +104,97 @@ void setClock(Clock clock) { @ApplicationModuleListener void onAiCompleted(AiAnalysisCompletedEvent event) { + // The AI analyzer computed the estimate before publishing, so it is only read here. + var cap = prepare(event.queryRequestId(), false); var query = load(event.queryRequestId(), "AiAnalysisCompletedEvent"); if (query != null) { - apply(query, queryDecisionEvaluator.evaluate(query, AiOutcome.COMPLETED, - event.riskLevel(), event.riskScore(), blockingRuleIds(query), clock)); + decide(query, AiOutcome.COMPLETED, event.riskLevel(), event.riskScore(), cap); } } @ApplicationModuleListener void onAiSkipped(AiAnalysisSkippedEvent event) { + // With AI off nothing has waited for the pre-flight estimate: an independent listener + // computes it, and routing would race it, so an estimated_rows / estimated_bytes_scanned + // policy would silently fail closed on an estimate that was about to exist (#941). + var cap = prepare(event.queryRequestId(), true); var query = load(event.queryRequestId(), "AiAnalysisSkippedEvent"); if (query != null) { - apply(query, queryDecisionEvaluator.evaluate(query, AiOutcome.SKIPPED, null, -1, - blockingRuleIds(query), clock)); + decide(query, AiOutcome.SKIPPED, null, -1, cap); } } @ApplicationModuleListener void onAiFailed(AiAnalysisFailedEvent event) { + var cap = prepare(event.queryRequestId(), false); var query = load(event.queryRequestId(), "AiAnalysisFailedEvent"); if (query != null) { - apply(query, queryDecisionEvaluator.evaluate(query, AiOutcome.FAILED, null, -1, - blockingRuleIds(query), clock)); + decide(query, AiOutcome.FAILED, null, -1, cap); + } + } + + /** + * Resolves the bytes-scanned cap (#941) and makes sure the pre-flight estimate exists — when AI + * was skipped, or when a cap applies — before the decision transaction reads anything. + * + *

It runs in its own transaction on purpose. The estimate write is a check-then-insert + * that races the independent estimate listener on a unique key, and it updates the + * version-checked {@code query_requests} row. Inside the decision transaction a lost race would + * mark that transaction rollback-only — and with it the transition, stranding the query in + * {@code PENDING_AI} — while a won race would leave the decision holding a stale query row. + * Here a lost race only fails this inner transaction; the winner's row is read afterwards. + */ + private AppliedBytesCap prepare(UUID queryRequestId, boolean alwaysEstimate) { + var template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + try { + return template.execute(status -> { + var query = queryRequestLookupService.findById(queryRequestId).orElse(null); + if (query == null || query.status() != QueryStatus.PENDING_AI) { + return null; + } + var cap = bytesScannedCapResolutionService + .resolve(query.datasourceId(), query.submittedByUserId()) + .orElse(null); + if (alwaysEstimate || cap != null) { + queryCostEstimateService.estimateSubmittedQuery(queryRequestId); + } + return cap; + }); + } catch (RuntimeException ex) { + // Typically the estimate listener won the insert race. The cap itself is re-resolved + // below; a missing estimate is a missing estimate, never an error. + log.warn("Pre-flight estimate preparation failed for query {}: {}", queryRequestId, + ex.getMessage()); + return resolveQuietly(queryRequestId); + } + } + + private AppliedBytesCap resolveQuietly(UUID queryRequestId) { + return queryRequestLookupService.findById(queryRequestId) + .flatMap(q -> bytesScannedCapResolutionService.resolve(q.datasourceId(), + q.submittedByUserId())) + .orElse(null); + } + + private void decide(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLevel riskLevel, + int riskScore, AppliedBytesCap cap) { + var bytesCap = cap == null ? null : BytesCapCheck.of(cap, estimatedBytes(query)); + var decision = queryDecisionEvaluator.evaluate(query, aiOutcome, riskLevel, riskScore, + blockingRuleIds(query), bytesCap, clock); + if (bytesCap != null) { + queryRequestStateService.recordBytesScannedCap(query.id(), bytesCap.limit(), + bytesCap.source(), bytesCap.outcome()); } + apply(query, decision); + } + + /** The persisted bytes estimate; absent, failed or unsupported all read as "none". */ + private Long estimatedBytes(QueryRequestSnapshot query) { + return queryEstimateLookupService.findByQueryRequestId(query.id()) + .filter(e -> !e.failed()) + .map(QueryEstimateSnapshot::estimatedBytesScanned) + .orElse(null); } private List blockingRuleIds(QueryRequestSnapshot query) { @@ -173,6 +260,12 @@ private void apply(QueryRequestSnapshot query, QueryDecision decision) { ? new QueryAutoApprovedEvent(query.id()) : new QueryReadyForReviewEvent(query.id())); } + case BYTES_CAP_REJECTED -> { + queryRequestStateService.transitionTo(query.id(), QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + eventPublisher.publishEvent(new QueryAutoRejectedEvent(query.id(), null, + bytesCapReason(decision.bytesCap()))); + } // A switch STATEMENT over an enum is not exhaustiveness-checked, so a new kind would // otherwise fall through silently and strand the query in PENDING_AI forever. default -> throw new IllegalStateException("Unhandled decision kind " + decision.kind()); @@ -180,6 +273,9 @@ private void apply(QueryRequestSnapshot query, QueryDecision decision) { if (decision.sqlReviewSuppression() != null) { auditSqlReviewBlocked(query, decision); } + if (decision.bytesCapChangedOutcome()) { + auditBytesCapEnforced(query, decision); + } // Logged after the fact: a line claiming a query was auto-approved must not outlive a // persistence call that then failed. if (match != null) { @@ -220,6 +316,46 @@ private void auditSqlReviewBlocked(QueryRequestSnapshot query, QueryDecision dec } } + /** + * One row per query whose outcome the bytes-scanned cap changed (#941) — a refusal, or an + * automatic approval turned into review. Swallow-and-log, like the SQL review row above. + */ + private void auditBytesCapEnforced(QueryRequestSnapshot query, QueryDecision decision) { + var cap = decision.bytesCap(); + var metadata = new LinkedHashMap(); + metadata.put("trigger", "bytes_scanned_cap"); + metadata.put("stage", "decision"); + metadata.put("limit", cap.limit()); + metadata.put("source", cap.source().name()); + if (cap.estimatedBytes() != null) { + metadata.put("estimated_bytes", cap.estimatedBytes()); + } + metadata.put("outcome", cap.outcome().name()); + if (decision.routingMatch() != null) { + metadata.put("matched_policy_id", decision.routingMatch().policyId()); + } + try { + auditLogService.record(new AuditEntry(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, + AuditResourceType.QUERY_REQUEST, query.id(), query.organizationId(), null, + metadata, null, null)); + } catch (RuntimeException ex) { + log.error("Audit write failed for QUERY_BYTES_SCANNED_CAP_ENFORCED on query {}", + query.id(), ex); + } + } + + /** Server-default locale for the same reason as {@link #grantReason}. */ + private String bytesCapReason(BytesCapCheck cap) { + var limit = ByteSizeFormat.format(cap.limit()); + if (cap.estimatedBytes() == null) { + return messageSource.getMessage("workflow.bytes_cap.rejected_no_estimate", + new Object[]{limit}, Locale.getDefault()); + } + return messageSource.getMessage("workflow.bytes_cap.rejected_exceeded", + new Object[]{ByteSizeFormat.format(cap.estimatedBytes()), limit}, + Locale.getDefault()); + } + /** * Rendered here rather than in the evaluator: there is no request locale on this asynchronous * path, so the reason falls back to the server default, and the evaluator stays free of diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java index 7b5d16544..f631a013c 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java @@ -64,7 +64,8 @@ public ConditionContext forLiveQuery(QueryRequestSnapshot query, RiskLevel riskL query.datasourceId(), query.id(), clock.instant()), behaviorAnomalyLookupService.hasActiveAnomaly(query.organizationId(), query.submittedByUserId(), query.datasourceId()), - estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed()); + estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed(), + estimate.bytesScanned()); } /** @@ -87,13 +88,14 @@ public ConditionContext forHistoricalRow(QueryCorpusRow row, ZoneId zone) { parsed.transactional(), row.submittedIp(), row.submittedUserAgent(), row.ciCdOrigin(), minutesSinceLastApproval(row.organizationId(), row.submittedByUserId(), row.datasourceId(), row.id(), row.createdAt()), - false, estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed()); + false, estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed(), + estimate.bytesScanned()); } /** * AF-624 pre-flight estimate signals. The estimate pipeline runs independently of AI analysis, * so whatever is persisted for the query is the signal; absent / unsupported / failed rows - * leave both fields null and the matching conditions fail closed. + * leave every field null and the matching conditions fail closed. * *

The replay arm reads it too: unlike membership or the anomaly flag, the estimate is a * persisted per-query fact, so dropping it would make an {@code estimated_rows} policy simulate @@ -102,15 +104,15 @@ public ConditionContext forHistoricalRow(QueryCorpusRow row, ZoneId zone) { private EstimateSignals estimateSignals(UUID queryRequestId) { var estimate = queryEstimateLookupService.findByQueryRequestId(queryRequestId).orElse(null); if (estimate == null || estimate.failed()) { - return new EstimateSignals(null, null); + return new EstimateSignals(null, null, null); } var rows = estimate.affectedRowCount() != null ? estimate.affectedRowCount() : estimate.estimatedRows(); - return new EstimateSignals(rows, estimate.scanType()); + return new EstimateSignals(rows, estimate.scanType(), estimate.estimatedBytesScanned()); } - private record EstimateSignals(Long rows, String scanType) { + private record EstimateSignals(Long rows, String scanType, Long bytesScanned) { } private String roleName(UUID userId) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java index 1d5db9596..0befada1b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java @@ -34,6 +34,7 @@ @JsonSubTypes.Type(value = ConditionNode.CiCdOrigin.class, name = "cicd_origin"), @JsonSubTypes.Type(value = ConditionNode.AnomalyDetected.class, name = "anomaly_detected"), @JsonSubTypes.Type(value = ConditionNode.EstimatedRows.class, name = "estimated_rows"), + @JsonSubTypes.Type(value = ConditionNode.EstimatedBytesScanned.class, name = "estimated_bytes_scanned"), @JsonSubTypes.Type(value = ConditionNode.ScanTypeMatches.class, name = "scan_type") }) interface ConditionNodeMixin { diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java index 339402976..69eb09520 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java @@ -53,6 +53,8 @@ public boolean matches(ConditionNode node, ConditionContext ctx) { case ConditionNode.AnomalyDetected c -> ctx.anomalyActive() == c.expected(); case ConditionNode.EstimatedRows c -> ctx.hasEstimateSignal() && c.operator().test(ctx.estimatedRows(), c.value()); + case ConditionNode.EstimatedBytesScanned c -> ctx.estimatedBytesScanned() != null + && c.operator().test(ctx.estimatedBytesScanned(), c.value()); case ConditionNode.ScanTypeMatches c -> matchesScanType(c, ctx); }; } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java index c41468144..2a8f69645 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java @@ -68,7 +68,8 @@ record EvaluatedContext(com.bablsoft.accessflow.core.api.QueryType queryType, String requesterIpAddress, String requesterUserAgent, boolean ciCdOrigin, Integer minutesSinceLastApproval, boolean anomalyActive, Long estimatedRows, String scanType, - List queryShapes, boolean shapesAnalyzed) { + List queryShapes, boolean shapesAnalyzed, + Long estimatedBytesScanned) { static EvaluatedContext from(ConditionContext context) { if (context == null) { @@ -82,7 +83,8 @@ static EvaluatedContext from(ConditionContext context) { context.requesterIpAddress(), context.requesterUserAgent(), context.ciCdOrigin(), context.minutesSinceLastApproval(), context.anomalyActive(), context.estimatedRows(), context.scanType(), - context.queryShapes().stream().sorted().toList(), context.shapesAnalyzed()); + context.queryShapes().stream().sorted().toList(), context.shapesAnalyzed(), + context.estimatedBytesScanned()); } } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java index 435ceb3d0..2ae42ef5c 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonRawValue; import com.bablsoft.accessflow.access.api.AccessGrantView; import com.bablsoft.accessflow.core.api.ApplicationNameSource; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.AiProviderType; import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.DecisionType; @@ -63,7 +65,9 @@ public record QueryDetailResponse( /** The calling application (#938); null when unknown. */ String applicationName, /** {@code API_KEY} (trustworthy) or {@code HEADER} (client-controlled). */ - ApplicationNameSource applicationNameSource) { + ApplicationNameSource applicationNameSource, + /** The bytes-scanned cap (#941) that applied to this query; null when none did. */ + BytesScannedCapDetail bytesScannedCap) { public static QueryDetailResponse from(QueryDetailView view) { return from(view, null, null); @@ -161,7 +165,21 @@ public static QueryDetailResponse from(QueryDetailView view, MatchedRoutingPolic view.onBehalfOfUserId() == null ? null : new OnBehalfOfRef(view.onBehalfOfUserId(), view.onBehalfOfEmail()), view.applicationName(), - view.applicationNameSource()); + view.applicationNameSource(), + BytesScannedCapDetail.from(view.bytesScannedCap())); + } + + /** + * The bytes-scanned cap (#941) recorded when the query left {@code PENDING_AI}: the limit, the + * configuration it came from, and how the estimate compared. + */ + public record BytesScannedCapDetail(long limit, BytesScannedCapSource source, + BytesScannedCapOutcome outcome) { + + static BytesScannedCapDetail from(QueryDetailView.BytesScannedCapDetail src) { + return src == null ? null + : new BytesScannedCapDetail(src.limit(), src.source(), src.outcome()); + } } /** A ticket auto-created in an external ticketing system for this query (AF-453). */ @@ -285,7 +303,8 @@ public record CostEstimateDetail( String unsupportedReason, boolean failed, String errorMessage, - Integer durationMs) { + Integer durationMs, + Long estimatedBytesScanned) { static CostEstimateDetail from(QueryDetailView.CostEstimateDetail src) { if (src == null) { @@ -305,7 +324,8 @@ static CostEstimateDetail from(QueryDetailView.CostEstimateDetail src) { src.unsupportedReason(), src.failed(), src.errorMessage(), - src.durationMs()); + src.durationMs(), + src.estimatedBytesScanned()); } } diff --git a/backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql b/backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql new file mode 100644 index 000000000..d9d287fae --- /dev/null +++ b/backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql @@ -0,0 +1,25 @@ +-- Bytes-scanned cost caps (#941). The warehouse engines' pre-flight scan estimate (AF-634) is now +-- persisted with the query's estimate, and an admin may cap it per datasource and per grant +-- (most restrictive wins; NULL = no cap). The cap is only configurable on engines that report a +-- bytes estimate (BigQuery, Snowflake, Databricks) — enforced by the service, not here. +ALTER TABLE query_estimates ADD COLUMN estimated_bytes_scanned BIGINT; + +CREATE TYPE bytes_cap_missing_estimate_action AS ENUM ('REQUIRE_REVIEW', 'REJECT'); +CREATE TYPE bytes_scanned_cap_source AS ENUM ('DATASOURCE', 'GRANT'); +CREATE TYPE bytes_scanned_cap_outcome AS ENUM ('WITHIN', 'EXCEEDED', 'NO_ESTIMATE_REVIEW', + 'NO_ESTIMATE_REJECTED'); + +ALTER TABLE datasources ADD COLUMN max_bytes_scanned_per_query BIGINT + CONSTRAINT chk_datasources_max_bytes_scanned_positive CHECK (max_bytes_scanned_per_query > 0); +ALTER TABLE datasources ADD COLUMN bytes_cap_missing_estimate bytes_cap_missing_estimate_action + NOT NULL DEFAULT 'REQUIRE_REVIEW'; + +ALTER TABLE datasource_user_permissions ADD COLUMN bytes_scanned_limit_override BIGINT + CONSTRAINT chk_dup_bytes_scanned_limit_positive CHECK (bytes_scanned_limit_override > 0); +ALTER TABLE datasource_group_permissions ADD COLUMN bytes_scanned_limit_override BIGINT + CONSTRAINT chk_dgp_bytes_scanned_limit_positive CHECK (bytes_scanned_limit_override > 0); + +-- Stamped when the query leaves PENDING_AI and a cap applied, so the detail view can say why. +ALTER TABLE query_requests ADD COLUMN bytes_scanned_cap BIGINT; +ALTER TABLE query_requests ADD COLUMN bytes_scanned_cap_source bytes_scanned_cap_source; +ALTER TABLE query_requests ADD COLUMN bytes_scanned_cap_outcome bytes_scanned_cap_outcome; diff --git a/backend/src/main/resources/i18n/messages.properties b/backend/src/main/resources/i18n/messages.properties index 799f0c02d..f28bc607e 100644 --- a/backend/src/main/resources/i18n/messages.properties +++ b/backend/src/main/resources/i18n/messages.properties @@ -1583,3 +1583,24 @@ validation.row_limit_table.required=Target table is required validation.row_limit_table.size=Target table must be at most 255 characters validation.row_limit_max_rows.required=Max rows is required validation.row_limit_max_rows.range=Max rows must be between 1 and 1,000,000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Bytes-scanned cap must be at least 1 +error.bytes_scanned_cap_not_supported=Bytes-scanned caps are not supported for {0} datasources; they are available only for engines that report a bytes estimate (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Refused before execution: the estimated scan of {0} exceeds the bytes-scanned cap of {1} +error.bytes_cap.no_estimate_unreviewed=Refused before execution: no bytes estimate is available under the bytes-scanned cap of {0}, and no one approved this request group, so the review the cap requires never happened +error.bytes_cap.no_estimate=Refused before execution: no bytes estimate is available and this datasource rejects queries without one under its bytes-scanned cap of {0} +workflow.bytes_cap.rejected_exceeded=Rejected by the bytes-scanned cap: the estimated scan of {0} exceeds the cap of {1} +workflow.bytes_cap.rejected_no_estimate=Rejected by the bytes-scanned cap of {0}: no bytes estimate is available and this datasource rejects queries without one +workflow.decision.bytes_cap.none=No bytes-scanned cap applies +workflow.decision.bytes_cap.unevaluated=A bytes-scanned cap of {0} applies; a simulated request has no estimate to compare +workflow.decision.bytes_cap.within=The estimated scan of {0} is within the bytes-scanned cap of {1} +workflow.decision.bytes_cap.exceeded=The estimated scan of {0} exceeds the bytes-scanned cap of {1} — the request is rejected +workflow.decision.bytes_cap.no_estimate_review=No bytes estimate is available under the bytes-scanned cap of {0} — the request cannot auto-approve +workflow.decision.bytes_cap.no_estimate_rejected=No bytes estimate is available under the bytes-scanned cap of {0} — the request is rejected +workflow.decision.routing.skipped_bytes_cap=Routing policies are not evaluated when the bytes-scanned cap rejects the request +workflow.decision.grant.skipped_bytes_cap=The grant fast path is not evaluated when the bytes-scanned cap rejects the request +workflow.decision.plan.skipped_bytes_cap=The review plan is not consulted when the bytes-scanned cap rejects the request +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=Routing policy "{0}" matched with action AUTO_APPROVE, suppressed because no bytes estimate is available under the bytes-scanned cap +workflow.decision.grant.suppressed_bytes_cap=Covered by access grant {0}, suppressed because no bytes estimate is available under the bytes-scanned cap +workflow.decision.plan.suppressed_bytes_cap=The review plan would have approved, suppressed because no bytes estimate is available under the bytes-scanned cap diff --git a/backend/src/main/resources/i18n/messages_de.properties b/backend/src/main/resources/i18n/messages_de.properties index b928a04d1..77c24973e 100644 --- a/backend/src/main/resources/i18n/messages_de.properties +++ b/backend/src/main/resources/i18n/messages_de.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=Zieltabelle ist erforderlich validation.row_limit_table.size=Zieltabelle darf höchstens 255 Zeichen lang sein validation.row_limit_max_rows.required=Maximale Zeilen sind erforderlich validation.row_limit_max_rows.range=Maximale Zeilen müssen zwischen 1 und 1.000.000 liegen + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Das Limit für gescannte Bytes muss mindestens 1 sein +error.bytes_scanned_cap_not_supported=Limits für gescannte Bytes werden für {0}-Datenquellen nicht unterstützt; sie gibt es nur für Engines mit Byte-Schätzung (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Vor der Ausführung abgelehnt: der geschätzte Scan von {0} überschreitet das Limit für gescannte Bytes von {1} +error.bytes_cap.no_estimate_unreviewed=Vor der Ausführung abgelehnt: unter dem Limit für gescannte Bytes von {0} liegt keine Byte-Schätzung vor, und niemand hat diese Anfragegruppe genehmigt, daher hat die vom Limit verlangte Prüfung nie stattgefunden +error.bytes_cap.no_estimate=Vor der Ausführung abgelehnt: es liegt keine Byte-Schätzung vor, und diese Datenquelle lehnt Abfragen ohne Schätzung unter ihrem Limit von {0} ab +workflow.bytes_cap.rejected_exceeded=Vom Limit für gescannte Bytes abgelehnt: der geschätzte Scan von {0} überschreitet das Limit von {1} +workflow.bytes_cap.rejected_no_estimate=Vom Limit für gescannte Bytes von {0} abgelehnt: es liegt keine Byte-Schätzung vor, und diese Datenquelle lehnt Abfragen ohne Schätzung ab +workflow.decision.bytes_cap.none=Es gilt kein Limit für gescannte Bytes +workflow.decision.bytes_cap.unevaluated=Es gilt ein Limit für gescannte Bytes von {0}; eine simulierte Anfrage hat keine Schätzung zum Vergleich +workflow.decision.bytes_cap.within=Der geschätzte Scan von {0} liegt innerhalb des Limits für gescannte Bytes von {1} +workflow.decision.bytes_cap.exceeded=Der geschätzte Scan von {0} überschreitet das Limit für gescannte Bytes von {1} — die Anfrage wird abgelehnt +workflow.decision.bytes_cap.no_estimate_review=Unter dem Limit für gescannte Bytes von {0} liegt keine Byte-Schätzung vor — die Anfrage kann nicht automatisch genehmigt werden +workflow.decision.bytes_cap.no_estimate_rejected=Unter dem Limit für gescannte Bytes von {0} liegt keine Byte-Schätzung vor — die Anfrage wird abgelehnt +workflow.decision.routing.skipped_bytes_cap=Routing-Richtlinien werden nicht ausgewertet, wenn das Limit für gescannte Bytes die Anfrage ablehnt +workflow.decision.grant.skipped_bytes_cap=Der Grant-Schnellpfad wird nicht ausgewertet, wenn das Limit für gescannte Bytes die Anfrage ablehnt +workflow.decision.plan.skipped_bytes_cap=Der Prüfplan wird nicht herangezogen, wenn das Limit für gescannte Bytes die Anfrage ablehnt +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=Routing-Richtlinie "{0}" hat mit Aktion AUTO_APPROVE gegriffen, unterdrückt, weil unter dem Limit für gescannte Bytes keine Byte-Schätzung vorliegt +workflow.decision.grant.suppressed_bytes_cap=Durch Zugriffsgewährung {0} abgedeckt, unterdrückt, weil unter dem Limit für gescannte Bytes keine Byte-Schätzung vorliegt +workflow.decision.plan.suppressed_bytes_cap=Der Prüfplan hätte genehmigt, unterdrückt, weil unter dem Limit für gescannte Bytes keine Byte-Schätzung vorliegt diff --git a/backend/src/main/resources/i18n/messages_es.properties b/backend/src/main/resources/i18n/messages_es.properties index 58b1e2ff0..de158e904 100644 --- a/backend/src/main/resources/i18n/messages_es.properties +++ b/backend/src/main/resources/i18n/messages_es.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=La tabla de destino es obligatoria validation.row_limit_table.size=La tabla de destino debe tener como máximo 255 caracteres validation.row_limit_max_rows.required=El máximo de filas es obligatorio validation.row_limit_max_rows.range=El máximo de filas debe estar entre 1 y 1.000.000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=El límite de bytes escaneados debe ser al menos 1 +error.bytes_scanned_cap_not_supported=Los límites de bytes escaneados no se admiten en fuentes de datos {0}; solo están disponibles para motores que informan una estimación de bytes (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Rechazada antes de ejecutarse: el escaneo estimado de {0} supera el límite de bytes escaneados de {1} +error.bytes_cap.no_estimate_unreviewed=Rechazada antes de ejecutarse: no hay estimación de bytes bajo el límite de bytes escaneados de {0} y nadie aprobó este grupo de solicitudes, así que la revisión que exige el límite nunca ocurrió +error.bytes_cap.no_estimate=Rechazada antes de ejecutarse: no hay estimación de bytes y esta fuente de datos rechaza las consultas sin ella bajo su límite de {0} +workflow.bytes_cap.rejected_exceeded=Rechazada por el límite de bytes escaneados: el escaneo estimado de {0} supera el límite de {1} +workflow.bytes_cap.rejected_no_estimate=Rechazada por el límite de bytes escaneados de {0}: no hay estimación de bytes y esta fuente de datos rechaza las consultas sin ella +workflow.decision.bytes_cap.none=No se aplica ningún límite de bytes escaneados +workflow.decision.bytes_cap.unevaluated=Se aplica un límite de bytes escaneados de {0}; una solicitud simulada no tiene estimación para comparar +workflow.decision.bytes_cap.within=El escaneo estimado de {0} está dentro del límite de bytes escaneados de {1} +workflow.decision.bytes_cap.exceeded=El escaneo estimado de {0} supera el límite de bytes escaneados de {1} — la solicitud se rechaza +workflow.decision.bytes_cap.no_estimate_review=No hay estimación de bytes bajo el límite de bytes escaneados de {0} — la solicitud no puede aprobarse automáticamente +workflow.decision.bytes_cap.no_estimate_rejected=No hay estimación de bytes bajo el límite de bytes escaneados de {0} — la solicitud se rechaza +workflow.decision.routing.skipped_bytes_cap=Las políticas de enrutamiento no se evalúan cuando el límite de bytes escaneados rechaza la solicitud +workflow.decision.grant.skipped_bytes_cap=La vía rápida por concesión no se evalúa cuando el límite de bytes escaneados rechaza la solicitud +workflow.decision.plan.skipped_bytes_cap=El plan de revisión no se consulta cuando el límite de bytes escaneados rechaza la solicitud +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=La política de enrutamiento "{0}" coincidió con la acción AUTO_APPROVE, suprimida porque no hay estimación de bytes bajo el límite de bytes escaneados +workflow.decision.grant.suppressed_bytes_cap=Cubierta por la concesión de acceso {0}, suprimida porque no hay estimación de bytes bajo el límite de bytes escaneados +workflow.decision.plan.suppressed_bytes_cap=El plan de revisión habría aprobado, suprimido porque no hay estimación de bytes bajo el límite de bytes escaneados diff --git a/backend/src/main/resources/i18n/messages_fr.properties b/backend/src/main/resources/i18n/messages_fr.properties index 25881f4a1..b072b3a59 100644 --- a/backend/src/main/resources/i18n/messages_fr.properties +++ b/backend/src/main/resources/i18n/messages_fr.properties @@ -1568,3 +1568,24 @@ validation.row_limit_table.required=La table cible est obligatoire validation.row_limit_table.size=La table cible ne doit pas dépasser 255 caractères validation.row_limit_max_rows.required=Le nombre maximal de lignes est obligatoire validation.row_limit_max_rows.range=Le nombre maximal de lignes doit être compris entre 1 et 1 000 000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=La limite d’octets analysés doit être d’au moins 1 +error.bytes_scanned_cap_not_supported=Les limites d’octets analysés ne sont pas prises en charge pour les sources de données {0} ; elles sont réservées aux moteurs qui fournissent une estimation en octets (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Refusée avant exécution : le volume analysé estimé de {0} dépasse la limite d’octets analysés de {1} +error.bytes_cap.no_estimate_unreviewed=Refusée avant exécution : aucune estimation en octets n’est disponible sous la limite d’octets analysés de {0}, et personne n’a approuvé ce groupe de demandes ; la revue exigée par la limite n’a donc jamais eu lieu +error.bytes_cap.no_estimate=Refusée avant exécution : aucune estimation en octets n’est disponible et cette source de données refuse les requêtes sans estimation sous sa limite de {0} +workflow.bytes_cap.rejected_exceeded=Rejetée par la limite d’octets analysés : le volume analysé estimé de {0} dépasse la limite de {1} +workflow.bytes_cap.rejected_no_estimate=Rejetée par la limite d’octets analysés de {0} : aucune estimation en octets n’est disponible et cette source de données refuse les requêtes sans estimation +workflow.decision.bytes_cap.none=Aucune limite d’octets analysés ne s’applique +workflow.decision.bytes_cap.unevaluated=Une limite d’octets analysés de {0} s’applique ; une demande simulée n’a pas d’estimation à comparer +workflow.decision.bytes_cap.within=Le volume analysé estimé de {0} respecte la limite d’octets analysés de {1} +workflow.decision.bytes_cap.exceeded=Le volume analysé estimé de {0} dépasse la limite d’octets analysés de {1} — la demande est rejetée +workflow.decision.bytes_cap.no_estimate_review=Aucune estimation en octets sous la limite d’octets analysés de {0} — la demande ne peut pas être approuvée automatiquement +workflow.decision.bytes_cap.no_estimate_rejected=Aucune estimation en octets sous la limite d’octets analysés de {0} — la demande est rejetée +workflow.decision.routing.skipped_bytes_cap=Les politiques de routage ne sont pas évaluées lorsque la limite d’octets analysés rejette la demande +workflow.decision.grant.skipped_bytes_cap=Le raccourci par autorisation n’est pas évalué lorsque la limite d’octets analysés rejette la demande +workflow.decision.plan.skipped_bytes_cap=Le plan de revue n’est pas consulté lorsque la limite d’octets analysés rejette la demande +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=La politique de routage "{0}" a correspondu avec l’action AUTO_APPROVE, neutralisée faute d’estimation en octets sous la limite d’octets analysés +workflow.decision.grant.suppressed_bytes_cap=Couverte par l’autorisation d’accès {0}, neutralisée faute d’estimation en octets sous la limite d’octets analysés +workflow.decision.plan.suppressed_bytes_cap=Le plan de revue aurait approuvé, neutralisé faute d’estimation en octets sous la limite d’octets analysés diff --git a/backend/src/main/resources/i18n/messages_hy.properties b/backend/src/main/resources/i18n/messages_hy.properties index fc51435bf..deb3b475b 100644 --- a/backend/src/main/resources/i18n/messages_hy.properties +++ b/backend/src/main/resources/i18n/messages_hy.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=Թիրախ աղյուսակը պարտադի validation.row_limit_table.size=Թիրախ աղյուսակը պետք է լինի առավելագույնը 255 նիշ validation.row_limit_max_rows.required=Տողերի առավելագույն քանակը պարտադիր է validation.row_limit_max_rows.range=Տողերի առավելագույն քանակը պետք է լինի 1-ից 1 000 000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Սկանավորված բայթերի սահմանաչափը պետք է լինի առնվազն 1 +error.bytes_scanned_cap_not_supported=Սկանավորված բայթերի սահմանաչափերը չեն աջակցվում {0} տվյալների աղբյուրների համար. դրանք հասանելի են միայն բայթերի գնահատական տրամադրող շարժիչների համար (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Մերժվել է կատարումից առաջ. {0} գնահատված սկանավորումը գերազանցում է {1} սկանավորված բայթերի սահմանաչափը +error.bytes_cap.no_estimate_unreviewed=Մերժվել է կատարումից առաջ. {0} սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա, և ոչ ոք չի հաստատել հարցումների այս խումբը, ուստի սահմանաչափի պահանջած ստուգումը տեղի չի ունեցել +error.bytes_cap.no_estimate=Մերժվել է կատարումից առաջ. բայթերի գնահատական չկա, և այս տվյալների աղբյուրը {0} սահմանաչափի ներքո մերժում է առանց գնահատականի հարցումները +workflow.bytes_cap.rejected_exceeded=Մերժվել է սկանավորված բայթերի սահմանաչափով. {0} գնահատված սկանավորումը գերազանցում է {1} սահմանաչափը +workflow.bytes_cap.rejected_no_estimate=Մերժվել է {0} սկանավորված բայթերի սահմանաչափով. բայթերի գնահատական չկա, և այս տվյալների աղբյուրը մերժում է առանց գնահատականի հարցումները +workflow.decision.bytes_cap.none=Սկանավորված բայթերի սահմանաչափ չի կիրառվում +workflow.decision.bytes_cap.unevaluated=Կիրառվում է {0} սկանավորված բայթերի սահմանաչափ. մոդելավորված հարցումը համեմատելու գնահատական չունի +workflow.decision.bytes_cap.within={0} գնահատված սկանավորումը {1} սկանավորված բայթերի սահմանաչափի մեջ է +workflow.decision.bytes_cap.exceeded={0} գնահատված սկանավորումը գերազանցում է {1} սկանավորված բայթերի սահմանաչափը — հարցումը մերժվում է +workflow.decision.bytes_cap.no_estimate_review={0} սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա — հարցումը չի կարող ավտոմատ հաստատվել +workflow.decision.bytes_cap.no_estimate_rejected={0} սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա — հարցումը մերժվում է +workflow.decision.routing.skipped_bytes_cap=Երթուղավորման քաղաքականությունները չեն գնահատվում, երբ սկանավորված բայթերի սահմանաչափը մերժում է հարցումը +workflow.decision.grant.skipped_bytes_cap=Թույլտվության արագ ուղին չի գնահատվում, երբ սկանավորված բայթերի սահմանաչափը մերժում է հարցումը +workflow.decision.plan.skipped_bytes_cap=Ստուգման պլանը չի դիտարկվում, երբ սկանավորված բայթերի սահմանաչափը մերժում է հարցումը +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=«{0}» երթուղավորման քաղաքականությունը համընկավ AUTO_APPROVE գործողությամբ, ճնշվել է, քանի որ սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա +workflow.decision.grant.suppressed_bytes_cap=Ծածկված է {0} մուտքի թույլտվությամբ, ճնշվել է, քանի որ սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա +workflow.decision.plan.suppressed_bytes_cap=Ստուգման պլանը կհաստատեր, ճնշվել է, քանի որ սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա diff --git a/backend/src/main/resources/i18n/messages_ru.properties b/backend/src/main/resources/i18n/messages_ru.properties index 13d462ce9..ccadf8a29 100644 --- a/backend/src/main/resources/i18n/messages_ru.properties +++ b/backend/src/main/resources/i18n/messages_ru.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=Целевая таблица обязате validation.row_limit_table.size=Целевая таблица не должна превышать 255 символов validation.row_limit_max_rows.required=Максимальное число строк обязательно validation.row_limit_max_rows.range=Максимальное число строк должно быть от 1 до 1 000 000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Лимит сканируемых байтов должен быть не менее 1 +error.bytes_scanned_cap_not_supported=Лимиты сканируемых байтов не поддерживаются для источников данных {0}; они доступны только для движков, сообщающих оценку в байтах (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Отклонено перед выполнением: оценочный объём сканирования {0} превышает лимит сканируемых байтов {1} +error.bytes_cap.no_estimate_unreviewed=Отклонено перед выполнением: при лимите сканируемых байтов {0} оценка в байтах недоступна, а эту группу запросов никто не одобрил, поэтому требуемая лимитом проверка не проводилась +error.bytes_cap.no_estimate=Отклонено перед выполнением: оценка в байтах недоступна, а этот источник данных отклоняет запросы без неё при лимите {0} +workflow.bytes_cap.rejected_exceeded=Отклонено лимитом сканируемых байтов: оценочный объём сканирования {0} превышает лимит {1} +workflow.bytes_cap.rejected_no_estimate=Отклонено лимитом сканируемых байтов {0}: оценка в байтах недоступна, а этот источник данных отклоняет запросы без неё +workflow.decision.bytes_cap.none=Лимит сканируемых байтов не применяется +workflow.decision.bytes_cap.unevaluated=Применяется лимит сканируемых байтов {0}; у смоделированного запроса нет оценки для сравнения +workflow.decision.bytes_cap.within=Оценочный объём сканирования {0} укладывается в лимит сканируемых байтов {1} +workflow.decision.bytes_cap.exceeded=Оценочный объём сканирования {0} превышает лимит сканируемых байтов {1} — запрос отклонён +workflow.decision.bytes_cap.no_estimate_review=При лимите сканируемых байтов {0} оценка в байтах недоступна — запрос не может быть одобрен автоматически +workflow.decision.bytes_cap.no_estimate_rejected=При лимите сканируемых байтов {0} оценка в байтах недоступна — запрос отклонён +workflow.decision.routing.skipped_bytes_cap=Политики маршрутизации не оцениваются, когда лимит сканируемых байтов отклоняет запрос +workflow.decision.grant.skipped_bytes_cap=Быстрый путь по выдаче доступа не оценивается, когда лимит сканируемых байтов отклоняет запрос +workflow.decision.plan.skipped_bytes_cap=План проверки не учитывается, когда лимит сканируемых байтов отклоняет запрос +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=Политика маршрутизации "{0}" сработала с действием AUTO_APPROVE, подавлено, так как при лимите сканируемых байтов оценка в байтах недоступна +workflow.decision.grant.suppressed_bytes_cap=Покрыт выдачей доступа {0}, подавлено, так как при лимите сканируемых байтов оценка в байтах недоступна +workflow.decision.plan.suppressed_bytes_cap=План проверки одобрил бы запрос, подавлено, так как при лимите сканируемых байтов оценка в байтах недоступна diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index f5fe78f60..9fff29495 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=目标表为必填项 validation.row_limit_table.size=目标表最多 255 个字符 validation.row_limit_max_rows.required=最大行数为必填项 validation.row_limit_max_rows.range=最大行数必须在 1 到 1,000,000 之间 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=扫描字节上限至少为 1 +error.bytes_scanned_cap_not_supported={0} 数据源不支持扫描字节上限；仅适用于提供字节估算的引擎（BigQuery、Snowflake、Databricks） +error.bytes_cap.exceeded=执行前被拒绝：预计扫描量 {0} 超过扫描字节上限 {1} +error.bytes_cap.no_estimate_unreviewed=执行前被拒绝：在 {0} 的扫描字节上限下没有可用的字节估算，且没有人批准此请求组，因此上限要求的审查从未进行 +error.bytes_cap.no_estimate=执行前被拒绝：没有可用的字节估算，且该数据源在 {0} 的扫描字节上限下拒绝没有估算的查询 +workflow.bytes_cap.rejected_exceeded=被扫描字节上限拒绝：预计扫描量 {0} 超过上限 {1} +workflow.bytes_cap.rejected_no_estimate=被 {0} 的扫描字节上限拒绝：没有可用的字节估算，且该数据源拒绝没有估算的查询 +workflow.decision.bytes_cap.none=未应用扫描字节上限 +workflow.decision.bytes_cap.unevaluated=适用 {0} 的扫描字节上限；模拟请求没有可比较的估算 +workflow.decision.bytes_cap.within=预计扫描量 {0} 在扫描字节上限 {1} 之内 +workflow.decision.bytes_cap.exceeded=预计扫描量 {0} 超过扫描字节上限 {1} — 请求被拒绝 +workflow.decision.bytes_cap.no_estimate_review=在 {0} 的扫描字节上限下没有可用的字节估算 — 请求无法自动批准 +workflow.decision.bytes_cap.no_estimate_rejected=在 {0} 的扫描字节上限下没有可用的字节估算 — 请求被拒绝 +workflow.decision.routing.skipped_bytes_cap=扫描字节上限拒绝请求时不评估路由策略 +workflow.decision.grant.skipped_bytes_cap=扫描字节上限拒绝请求时不评估授权快速通道 +workflow.decision.plan.skipped_bytes_cap=扫描字节上限拒绝请求时不参考审查计划 +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=路由策略"{0}"以 AUTO_APPROVE 动作匹配，但因在扫描字节上限下没有可用的字节估算而被抑制 +workflow.decision.grant.suppressed_bytes_cap=由访问授权 {0} 覆盖，但因在扫描字节上限下没有可用的字节估算而被抑制 +workflow.decision.plan.suppressed_bytes_cap=审查计划本会批准，但因在扫描字节上限下没有可用的字节估算而被抑制 diff --git a/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java b/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java index 27bad6809..85696530a 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java @@ -69,7 +69,7 @@ private AccessGrantRequestEntity approved() { private DatasourcePermissionView granted() { return new DatasourcePermissionView(newPermissionId, datasourceId, requesterId, "u@x.io", - "U", true, false, false, false, null, List.of("public"), null, null, null, List.of(), List.of(), + "U", true, false, false, false, null, null, List.of("public"), null, null, null, List.of(), List.of(), List.of(), Instant.now().plusSeconds(3600), approverId, Instant.now()); } @@ -155,6 +155,25 @@ void materialiseCarriesTheReplacedRowsDenialsOntoTheNewGrant() { .containsExactly(com.bablsoft.accessflow.core.api.QueryShape.JOIN); } + @Test + void materialiseCarriesTheReplacedRowsBytesScannedCapSoAJitApprovalNeverWidensIt() { + when(requestRepository.findById(requestId)).thenReturn(Optional.of(approved())); + var existing = new DatasourceUserPermissionView(UUID.randomUUID(), requesterId, + datasourceId, true, false, false, false, null, null, null, null, null, null, null, + null, 4_000L, Instant.now().plusSeconds(60)); + when(permissionLookupService.findDirectFor(requesterId, datasourceId)) + .thenReturn(Optional.of(existing)); + when(datasourceAdminService.grantPermission(any(), any(), any(), any())) + .thenReturn(granted()); + + materializer.materialize(requestId, approverId); + + var captor = ArgumentCaptor.forClass(CreatePermissionCommand.class); + verify(datasourceAdminService).grantPermission(eq(datasourceId), eq(organizationId), + eq(approverId), captor.capture()); + assertThat(captor.getValue().bytesScannedLimitOverride()).isEqualTo(4_000L); + } + @Test void materialiseWithoutAnExistingRowCarriesNoDenials() { when(requestRepository.findById(requestId)).thenReturn(Optional.of(approved())); diff --git a/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java index 4aa4ac2a6..6f406ce6b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java @@ -109,7 +109,7 @@ private void givenDatasourceGrant(List allowedTables, Instant grantedAt) .thenReturn(List.of(new DatasourceRef(DATASOURCE, "analytics"))); when(datasourceAdminService.listPermissions(DATASOURCE, ORG)).thenReturn(List.of( new DatasourcePermissionView(PERMISSION, DATASOURCE, USER, "dev@example.test", - "Dev", true, false, false, false, null, List.of(), allowedTables, List.of(), null, List.of(), List.of(), + "Dev", true, false, false, false, null, null, List.of(), allowedTables, List.of(), null, List.of(), List.of(), List.of(), null, UUID.randomUUID(), grantedAt))); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java index 3a7fa7b5e..a0b918253 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java @@ -251,6 +251,28 @@ void analyzeSubmittedQueryPersistsResultAndPublishesCompleted() { .isEqualTo(50.0); } + @Test + void theCostEstimateContextCarriesTheWarehouseBytesEstimate() { + var snapshot = new QueryRequestSnapshot(queryRequestId, datasourceId, organizationId, userId, + "SELECT 1", QueryType.SELECT, false, QueryStatus.PENDING_AI, null, null, null, false); + when(queryRequestLookupService.findById(queryRequestId)).thenReturn(Optional.of(snapshot)); + when(datasourceLookupService.findById(datasourceId)).thenReturn(Optional.of(descriptor(DbType.MYSQL))); + when(datasourceAdminService.introspectSchemaForSystem(datasourceId, organizationId)).thenReturn(schemaView()); + when(queryCostEstimateService.estimateSubmittedQuery(queryRequestId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), + queryRequestId, "bigquery", QueryType.SELECT, true, null, null, null, null, + 2_500_000_000L, null, null, null, false, null, 5, java.time.Instant.now()))); + var costContext = ArgumentCaptor.forClass(String.class); + when(strategy.analyze(eq("SELECT 1"), eq(DbType.MYSQL), any(), costContext.capture(), any(), + eq(aiConfigId))).thenReturn(sampleResult()); + when(aiAnalysisPersistenceService.persist(eq(queryRequestId), any())).thenReturn(UUID.randomUUID()); + + service.analyzeSubmittedQuery(queryRequestId); + + assertThat(costContext.getValue()) + .isEqualTo("The warehouse estimates this query will scan 2.5 GB (2500000000 B)."); + } + @Test void analyzeSubmittedQueryPersistsPerModelBreakdown() { var snapshot = new QueryRequestSnapshot(queryRequestId, datasourceId, organizationId, userId, diff --git a/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java index 22fbde2d5..a397125ed 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java @@ -130,7 +130,7 @@ private UserEntity user(String name, UserRoleType role) { private void grant(UserEntity subject) { datasourceAdminService.grantPermission(datasource.getId(), organization.getId(), admin.getId(), new CreatePermissionCommand(subject.getId(), true, false, false, - false, null, List.of("public"), null, null, null, null, null, List.of(), null, null)); + false, null, null, List.of("public"), null, null, null, null, null, List.of(), null, null)); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java index e3d149031..92b482f37 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java @@ -79,7 +79,7 @@ private AttestationCampaignEntity scheduledDatasourceCampaign() { private DatasourcePermissionView permission(UUID userId) { return new DatasourcePermissionView(UUID.randomUUID(), datasourceId, userId, userId + "@example.com", "User", true, false, false, false, null, - List.of("public"), List.of(), List.of(), null, List.of(), List.of(), List.of(), null, UUID.randomUUID(), Instant.now()); + null, List.of("public"), List.of(), List.of(), null, List.of(), List.of(), List.of(), null, UUID.randomUUID(), Instant.now()); } @Test @@ -113,7 +113,7 @@ void openSnapshotsTheGrantsTableAndSchemaDenials() { when(datasourceAdminService.listPermissions(datasourceId, orgId)) .thenReturn(List.of(new DatasourcePermissionView(UUID.randomUUID(), datasourceId, userId, "u@example.com", "User", true, false, false, false, null, - List.of("crm"), List.of(), List.of(), List.of(), List.of("hr"), + 8_000L, List.of("crm"), List.of(), List.of(), List.of(), List.of("hr"), List.of("crm.salary"), List.of(com.bablsoft.accessflow.core.api.QueryShape.GROUP_BY), null, UUID.randomUUID(), Instant.now()))); @@ -129,6 +129,7 @@ void openSnapshotsTheGrantsTableAndSchemaDenials() { assertThat(snapshot.get("denied_tables").get(0).asString()).isEqualTo("crm.salary"); assertThat(snapshot.get("denied_tables")).hasSize(1); assertThat(snapshot.get("denied_shapes").get(0).asString()).isEqualTo("GROUP_BY"); + assertThat(snapshot.get("bytes_scanned_limit_override").asLong()).isEqualTo(8_000L); } @Test @@ -138,7 +139,7 @@ void openSnapshotsEmptyDenialArraysForAGrantWithoutDenials() { when(datasourceLookupService.findRef(datasourceId)) .thenReturn(Optional.of(new DatasourceRef(datasourceId, "Production"))); var view = new DatasourcePermissionView(UUID.randomUUID(), datasourceId, UUID.randomUUID(), - "u@example.com", "User", true, false, false, false, null, null, null, null, null, + "u@example.com", "User", true, false, false, false, null, null, null, null, null, null, null, null, null, null, null, null); when(datasourceAdminService.listPermissions(datasourceId, orgId)).thenReturn(List.of(view)); when(itemRepository.existsByCampaignIdAndPermissionId(any(), any())).thenReturn(false); @@ -152,6 +153,7 @@ void openSnapshotsEmptyDenialArraysForAGrantWithoutDenials() { assertThat(snapshot.get("denied_schemas")).isEmpty(); assertThat(snapshot.get("denied_tables")).isEmpty(); assertThat(snapshot.get("denied_shapes")).isEmpty(); + assertThat(snapshot.get("bytes_scanned_limit_override").isNull()).isTrue(); } /** diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java new file mode 100644 index 000000000..a5151fca9 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java @@ -0,0 +1,90 @@ +package com.bablsoft.accessflow.core.api; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class BytesScannedCapRecordsTest { + + @Test + void anEstimateAboveTheCapExceedsIt() { + var cap = new AppliedBytesCap(100, BytesScannedCapSource.DATASOURCE, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + assertThat(cap.check(101L)).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + assertThat(cap.check(100L)).isEqualTo(BytesScannedCapOutcome.WITHIN); + assertThat(cap.check(0L)).isEqualTo(BytesScannedCapOutcome.WITHIN); + } + + @Test + void aMissingEstimateFollowsTheDatasourcePolicy() { + var review = new AppliedBytesCap(100, BytesScannedCapSource.GRANT, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var reject = new AppliedBytesCap(100, BytesScannedCapSource.GRANT, + BytesCapMissingEstimateAction.REJECT); + + assertThat(review.check(null)).isEqualTo(BytesScannedCapOutcome.NO_ESTIMATE_REVIEW); + assertThat(reject.check(null)).isEqualTo(BytesScannedCapOutcome.NO_ESTIMATE_REJECTED); + } + + @Test + void anAbsentPolicyDefaultsToRequireReview() { + var cap = new AppliedBytesCap(1, BytesScannedCapSource.DATASOURCE, null); + + assertThat(cap.missingEstimate()).isEqualTo(BytesCapMissingEstimateAction.REQUIRE_REVIEW); + } + + @Test + void aCapMustBePositiveAndNamed() { + assertThatThrownBy(() -> new AppliedBytesCap(0, BytesScannedCapSource.DATASOURCE, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new AppliedBytesCap(1, null, null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void onlyTheTwoRefusingOutcomesReject() { + assertThat(BytesScannedCapOutcome.EXCEEDED.rejects()).isTrue(); + assertThat(BytesScannedCapOutcome.NO_ESTIMATE_REJECTED.rejects()).isTrue(); + assertThat(BytesScannedCapOutcome.WITHIN.rejects()).isFalse(); + assertThat(BytesScannedCapOutcome.NO_ESTIMATE_REVIEW.rejects()).isFalse(); + } + + @Test + void onlyTheBytesReportingWarehousesSupportACap() { + assertThat(BytesScannedCapSupport.supports(DbType.BIGQUERY)).isTrue(); + assertThat(BytesScannedCapSupport.supports(DbType.SNOWFLAKE)).isTrue(); + assertThat(BytesScannedCapSupport.supports(DbType.DATABRICKS)).isTrue(); + assertThat(BytesScannedCapSupport.supports(DbType.POSTGRESQL)).isFalse(); + assertThat(BytesScannedCapSupport.supports(DbType.MONGODB)).isFalse(); + assertThat(BytesScannedCapSupport.supports(null)).isFalse(); + assertThat(BytesScannedCapSupport.supportedTypes()).hasSize(3); + } + + @Test + void theExceptionsCarryWhatTheirHandlersRender() { + var notSupported = new BytesScannedCapNotSupportedException(DbType.POSTGRESQL); + assertThat(notSupported.dbType()).isEqualTo(DbType.POSTGRESQL); + assertThat(notSupported).hasMessageContaining("POSTGRESQL"); + + var cap = new AppliedBytesCap(10, BytesScannedCapSource.GRANT, null); + var exceeded = new BytesScannedCapExceededException("too big", cap, 20L, + BytesScannedCapOutcome.EXCEEDED); + assertThat(exceeded).hasMessage("too big"); + assertThat(exceeded.cap()).isEqualTo(cap); + assertThat(exceeded.estimatedBytes()).isEqualTo(20L); + assertThat(exceeded.outcome()).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + } + + @Test + void byteSizesRenderInDecimalUnitsWithTheExactFigure() { + assertThat(ByteSizeFormat.format(0)).isEqualTo("0 B"); + assertThat(ByteSizeFormat.format(999)).isEqualTo("999 B"); + assertThat(ByteSizeFormat.format(1_000)).isEqualTo("1 KB (1000 B)"); + assertThat(ByteSizeFormat.format(1_500_000_000_000L)) + .isEqualTo("1.5 TB (1500000000000 B)"); + assertThat(ByteSizeFormat.format(1_234_567_890L)).isEqualTo("1.23 GB (1234567890 B)"); + assertThat(ByteSizeFormat.format(Long.MAX_VALUE)).startsWith("9.22 EB"); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java index db0c4ca80..85a450973 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java @@ -830,6 +830,121 @@ void updateClearEnvironmentUnsetsItAndAnExplicitValueWins() { assertThat(entity.getEnvironment()).isEqualTo(DatasourceEnvironment.DEVELOPMENT); } + // ── Bytes-scanned cap (#941) ────────────────────────────────────────────── + + @Test + void createStoresABytesScannedCapOnAWarehouse() { + var org = new OrganizationEntity(); + org.setId(orgId); + when(organizationRepository.getReferenceById(orgId)).thenReturn(org); + when(encryptionService.encrypt("pw")).thenReturn("ENC(pw)"); + when(engineCatalog.isEngineManaged(DbType.SNOWFLAKE)).thenReturn(true); + when(datasourceRepository.save(any(DatasourceEntity.class))) + .thenAnswer(inv -> inv.getArgument(0)); + + var result = service.create(bytesCapCreateCommand(DbType.SNOWFLAKE, 1_000_000L, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT)); + + assertThat(result.maxBytesScannedPerQuery()).isEqualTo(1_000_000L); + assertThat(result.bytesCapMissingEstimate()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + } + + @Test + void createWithoutACapDefaultsToRequireReview() { + var org = new OrganizationEntity(); + org.setId(orgId); + when(organizationRepository.getReferenceById(orgId)).thenReturn(org); + when(encryptionService.encrypt("pw")).thenReturn("ENC(pw)"); + when(datasourceRepository.save(any(DatasourceEntity.class))) + .thenAnswer(inv -> inv.getArgument(0)); + + var result = service.create(createCommand("Prod", null)); + + assertThat(result.maxBytesScannedPerQuery()).isNull(); + assertThat(result.bytesCapMissingEstimate()).isEqualTo( + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + } + + @Test + void createRefusesABytesScannedCapOnAnEngineWithoutABytesEstimate() { + var org = new OrganizationEntity(); + org.setId(orgId); + when(organizationRepository.getReferenceById(orgId)).thenReturn(org); + when(encryptionService.encrypt("pw")).thenReturn("ENC(pw)"); + + assertThatThrownBy(() -> service.create(bytesCapCreateCommand(DbType.POSTGRESQL, 10L, null))) + .isInstanceOf(com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException.class); + verify(datasourceRepository, never()).save(any()); + } + + @Test + void updateSetsClearsAndGatesTheBytesScannedCap() { + var entity = buildDatasource(datasourceId, orgId, "Wh"); + entity.setDbType(DbType.BIGQUERY); + when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.of(entity)); + + service.update(datasourceId, orgId, bytesCapUpdateCommand(500L, null, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT)); + assertThat(entity.getMaxBytesScannedPerQuery()).isEqualTo(500L); + assertThat(entity.getBytesCapMissingEstimate()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + + service.update(datasourceId, orgId, bytesCapUpdateCommand(null, null, null)); + assertThat(entity.getMaxBytesScannedPerQuery()).isEqualTo(500L); + + service.update(datasourceId, orgId, bytesCapUpdateCommand(null, true, null)); + assertThat(entity.getMaxBytesScannedPerQuery()).isNull(); + + entity.setDbType(DbType.MYSQL); + assertThatThrownBy(() -> service.update(datasourceId, orgId, + bytesCapUpdateCommand(10L, null, null))) + .isInstanceOf(com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException.class); + } + + @Test + void grantPermissionStoresABytesScannedOverrideOnlyOnAWarehouse() { + stubGrantableUser(DbType.DATABRICKS); + var saved = ArgumentCaptor.forClass(DatasourceUserPermissionEntity.class); + when(permissionRepository.save(saved.capture())).thenAnswer(inv -> inv.getArgument(0)); + + var view = service.grantPermission(datasourceId, orgId, adminId, bytesCapPermission(900L)); + + assertThat(saved.getValue().getBytesScannedLimitOverride()).isEqualTo(900L); + assertThat(view.bytesScannedLimitOverride()).isEqualTo(900L); + } + + @Test + void grantPermissionRefusesABytesScannedOverrideOnARelationalDatasource() { + stubGrantableUser(DbType.POSTGRESQL); + + assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, + bytesCapPermission(900L))) + .isInstanceOf(com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException.class); + verify(permissionRepository, never()).save(any()); + } + + private CreateDatasourceCommand bytesCapCreateCommand( + DbType dbType, Long cap, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + return new CreateDatasourceCommand(orgId, "Wh", dbType, "wh.example.com", 443, "ANALYTICS", + "svc", "pw", SslMode.REQUIRE, null, null, null, null, null, false, null, null, + null, null, null, null, null, null, null, null, null, null, cap, missing); + } + + private static UpdateDatasourceCommand bytesCapUpdateCommand( + Long cap, Boolean clear, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + return new UpdateDatasourceCommand(null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, cap, clear, missing); + } + + private CreatePermissionCommand bytesCapPermission(Long override) { + return new CreatePermissionCommand(userId, true, false, false, false, null, override, null, + null, null, null, null, null, null, null, null); + } + private CreateDatasourceCommand createCommand(String name, DatasourceEnvironment environment) { return new CreateDatasourceCommand(orgId, name, DbType.POSTGRESQL, "db", 5432, "appdb", "svc", "pw", SslMode.DISABLE, null, null, null, null, null, false, null, null, @@ -1076,7 +1191,7 @@ void grantPermissionRejectsUserFromDifferentOrg() { user.setOrganization(otherOrg); when(userRepository.findById(userId)).thenReturn(Optional.of(user)); - var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, + var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null, null); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(IllegalDatasourcePermissionException.class); @@ -1088,7 +1203,7 @@ void grantPermissionRejectsUnknownUser() { when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.of(entity)); when(userRepository.findById(userId)).thenReturn(Optional.empty()); - var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, + var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null, null); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(IllegalDatasourcePermissionException.class); @@ -1107,7 +1222,7 @@ void grantPermissionRejectsDuplicate() { when(permissionRepository.existsByUser_IdAndDatasource_Id(userId, datasourceId)) .thenReturn(true); - var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, + var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null, null); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(DatasourcePermissionAlreadyExistsException.class); @@ -1135,7 +1250,7 @@ void grantPermissionPersistsAndReturnsView() { .thenAnswer(inv -> inv.getArgument(0)); var command = new CreatePermissionCommand(userId, true, true, false, true, 500, - List.of("public"), List.of("orders"), List.of("public.orders.ssn"), null, null, null, List.of(), null, null); + null, List.of("public"), List.of("orders"), List.of("public.orders.ssn"), null, null, null, List.of(), null, null); var view = service.grantPermission(datasourceId, orgId, adminId, command); // An admin-created row has no originating JIT request (#969). @@ -1174,7 +1289,7 @@ void grantPermissionStampsTheOriginatingAccessRequest() { var accessGrantRequestId = UUID.randomUUID(); service.grantPermission(datasourceId, orgId, adminId, new CreatePermissionCommand(userId, - true, false, false, false, null, null, null, null, null, null, null, + true, false, false, false, null, null, null, null, null, null, null, null, List.of(), Instant.now().plusSeconds(3600), accessGrantRequestId)); @@ -1240,7 +1355,7 @@ void grantPermissionPersistsNormalizedDeniedSchemasAndTables() { var view = service.grantPermission(datasourceId, orgId, adminId, new CreatePermissionCommand(userId, true, false, false, false, null, - List.of("crm"), null, null, null, List.of(" \"HR\" ", "hr"), + null, List.of("crm"), null, null, null, List.of(" \"HR\" ", "hr"), List.of("CRM.Salary", "`crm`.`salary`", "bonus"), List.of(), null, null)); assertThat(saved.getValue().getDeniedSchemas()).containsExactly("hr"); @@ -1254,12 +1369,12 @@ void grantPermissionRefusesADeniedEntryThatCouldNeverMatch() { stubGrantableUser(DbType.POSTGRESQL); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, - new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, List.of("analytics.hr"), null, List.of(), null, null))) .isInstanceOf(IllegalDatasourcePermissionException.class) .hasMessageContaining("denied_schemas"); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, - new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, List.of("sal*"), List.of(), null, null))) .isInstanceOf(IllegalDatasourcePermissionException.class) .hasMessageContaining("denied_tables"); @@ -1273,7 +1388,7 @@ void grantPermissionStoresNoDeniedSchemasOrTablesWhenEmptyOrBlank() { when(permissionRepository.save(saved.capture())).thenAnswer(inv -> inv.getArgument(0)); var view = service.grantPermission(datasourceId, orgId, adminId, - new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, List.of(), List.of(" "), List.of(), null, null)); assertThat(saved.getValue().getDeniedSchemas()).isNull(); @@ -1321,7 +1436,7 @@ void grantPermissionRejectsDeniedShapesOnAnEngineManagedDatasource() { } private CreatePermissionCommand shapesCommand(List deniedShapes) { - return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, deniedShapes, null, null); } @@ -1340,7 +1455,7 @@ private void stubGrantableUser(DbType dbType) { } private CreatePermissionCommand deniedCommand(List deniedColumns) { - return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, deniedColumns, null, null, List.of(), null, null); } @@ -1408,7 +1523,7 @@ void grantGroupPermissionPersistsAndReturnsView() { .thenAnswer(inv -> inv.getArgument(0)); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, true, false, false, null, List.of("public"), null, null, null, null, null, List.of(), null); + groupId, true, true, false, false, null, null, List.of("public"), null, null, null, null, null, List.of(), null); var view = service.grantGroupPermission(datasourceId, orgId, adminId, command); assertThat(view.groupId()).isEqualTo(groupId); @@ -1439,7 +1554,7 @@ void grantGroupPermissionPersistsNormalizedDeniedSchemasAndTables() { var view = service.grantGroupPermission(datasourceId, orgId, adminId, new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, + groupId, true, false, false, false, null, null, null, null, null, null, List.of("Audit"), List.of("CRM.Salary", " "), List.of(), null)); assertThat(saved.getValue().getDeniedSchemas()).containsExactly("audit"); @@ -1464,7 +1579,7 @@ void grantGroupPermissionPersistsDeniedShapesAndRefusesThemOnAnEngineManagedData com.bablsoft.accessflow.core.internal.persistence.entity.DatasourceGroupPermissionEntity.class); when(groupPermissionRepository.save(saved.capture())).thenAnswer(inv -> inv.getArgument(0)); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, null, null, + groupId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(QueryShape.HAVING, QueryShape.GROUP_BY), null); var view = service.grantGroupPermission(datasourceId, orgId, adminId, command); @@ -1496,7 +1611,7 @@ void grantGroupPermissionStoresNoDeniedSchemasOrTablesWhenAbsent() { var view = service.grantGroupPermission(datasourceId, orgId, adminId, new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, + groupId, true, false, false, false, null, null, null, null, null, null, List.of(), null, List.of(), null)); assertThat(saved.getValue().getDeniedSchemas()).isNull(); @@ -1516,7 +1631,7 @@ void grantGroupPermissionRejectsDuplicate() { .thenReturn(true); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, null, null, List.of(), null); + groupId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null); assertThatThrownBy(() -> service.grantGroupPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(com.bablsoft.accessflow.core.api.DatasourceGroupPermissionAlreadyExistsException.class); } @@ -1530,7 +1645,7 @@ void grantGroupPermissionRejectsUnknownGroup() { .thenThrow(new com.bablsoft.accessflow.core.api.UserGroupNotFoundException(groupId)); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, null, null, List.of(), null); + groupId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null); assertThatThrownBy(() -> service.grantGroupPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(com.bablsoft.accessflow.core.api.UserGroupNotFoundException.class); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java new file mode 100644 index 000000000..d801f3b04 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java @@ -0,0 +1,126 @@ +package com.bablsoft.accessflow.core.internal; + +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionLookupService; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionView; +import com.bablsoft.accessflow.core.internal.persistence.entity.DatasourceEntity; +import com.bablsoft.accessflow.core.internal.persistence.repo.DatasourceRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DefaultBytesScannedCapResolutionServiceTest { + + @Mock DatasourceRepository datasourceRepository; + @Mock DatasourceUserPermissionLookupService permissionLookupService; + @InjectMocks DefaultBytesScannedCapResolutionService service; + + private final UUID datasourceId = UUID.randomUUID(); + private final UUID userId = UUID.randomUUID(); + + @Test + void noCapAnywhereResolvesToNothing() { + givenDatasource(null, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(null); + + assertThat(service.resolve(datasourceId, userId)).isEmpty(); + } + + @Test + void anUnknownDatasourceResolvesToNothing() { + when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.empty()); + + assertThat(service.resolve(datasourceId, userId)).isEmpty(); + verifyNoInteractions(permissionLookupService); + } + + @Test + void theDatasourceCapAppliesWhenNoGrantSetsOne() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REJECT); + givenGrant(null); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(1_000L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.DATASOURCE); + assertThat(cap.missingEstimate()).isEqualTo(BytesCapMissingEstimateAction.REJECT); + } + + @Test + void aTighterGrantWins() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(400L); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(400L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.GRANT); + } + + @Test + void aLooserGrantNeverWidensTheDatasourceCap() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(5_000L); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(1_000L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.DATASOURCE); + } + + @Test + void aTieIsAttributedToTheDatasource() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(1_000L); + + assertThat(service.resolve(datasourceId, userId).orElseThrow().source()) + .isEqualTo(BytesScannedCapSource.DATASOURCE); + } + + @Test + void aGrantCapAppliesOnAnUncappedDatasourceWithItsMissingEstimatePolicy() { + givenDatasource(null, BytesCapMissingEstimateAction.REJECT); + givenGrant(300L); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(300L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.GRANT); + assertThat(cap.missingEstimate()).isEqualTo(BytesCapMissingEstimateAction.REJECT); + } + + @Test + void anAnonymousLookupOnlyReadsTheDatasource() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + assertThat(service.resolve(datasourceId, null)).isPresent(); + verifyNoInteractions(permissionLookupService); + } + + private void givenDatasource(Long cap, BytesCapMissingEstimateAction missing) { + var entity = new DatasourceEntity(); + entity.setId(datasourceId); + entity.setMaxBytesScannedPerQuery(cap); + entity.setBytesCapMissingEstimate(missing); + when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.of(entity)); + } + + private void givenGrant(Long override) { + var view = new DatasourceUserPermissionView(UUID.randomUUID(), userId, datasourceId, true, + false, false, false, List.of(), List.of(), List.of(), List.of(), List.of(), + List.of(), List.of(), null, override, null); + when(permissionLookupService.findFor(userId, datasourceId)).thenReturn(Optional.of(view)); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java index 4954d9034..142c8cee5 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java @@ -544,6 +544,51 @@ void findDirectForAndContributionsCarryTheRowLimitOverride() { .isEqualTo(75); } + @Test + void findForTakesTheSmallestBytesScannedCapAcrossGrants() { + var userId = UUID.randomUUID(); + var datasourceId = UUID.randomUUID(); + var groupId = UUID.randomUUID(); + var direct = newPermission(UUID.randomUUID(), userId, datasourceId); + direct.setCanRead(true); + direct.setBytesScannedLimitOverride(5_000L); + var group = newGroupPermission(groupId, datasourceId); + group.setCanRead(true); + group.setBytesScannedLimitOverride(2_000L); + when(permissionRepository.findByUser_IdAndDatasource_Id(userId, datasourceId)) + .thenReturn(Optional.of(direct)); + when(membershipRepository.findGroupIdsForUser(userId)).thenReturn(List.of(groupId)); + when(groupPermissionRepository.findAllByGroup_IdIn(List.of(groupId))) + .thenReturn(List.of(group)); + + assertThat(service.findFor(userId, datasourceId).orElseThrow().bytesScannedLimitOverride()) + .isEqualTo(2_000L); + assertThat(service.findDirectFor(userId, datasourceId).orElseThrow() + .bytesScannedLimitOverride()).isEqualTo(5_000L); + assertThat(service.findContributions(userId, datasourceId)) + .extracting(c -> c.bytesScannedLimitOverride()).containsExactly(5_000L, 2_000L); + } + + @Test + void findForBytesScannedCapIsNullWhenNoGrantSetsOneAndOneSetOneWins() { + var userId = UUID.randomUUID(); + var datasourceId = UUID.randomUUID(); + var groupId = UUID.randomUUID(); + var direct = newPermission(UUID.randomUUID(), userId, datasourceId); + direct.setCanRead(true); + var group = newGroupPermission(groupId, datasourceId); + group.setCanRead(true); + group.setBytesScannedLimitOverride(700L); + when(permissionRepository.findByUser_IdAndDatasource_Id(userId, datasourceId)) + .thenReturn(Optional.of(direct)); + when(membershipRepository.findGroupIdsForUser(userId)).thenReturn(List.of(groupId)); + when(groupPermissionRepository.findAllByGroup_IdIn(List.of(groupId))) + .thenReturn(List.of(group)); + + assertThat(service.findFor(userId, datasourceId).orElseThrow().bytesScannedLimitOverride()) + .isEqualTo(700L); + } + private DatasourceUserPermissionView mergeRowLimits( Integer directLimit, Integer groupLimit) { var userId = UUID.randomUUID(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java index 7ef7e6718..460004424 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java @@ -33,7 +33,7 @@ class DefaultQueryEstimateServiceTest { private static PersistQueryEstimateCommand command() { return new PersistQueryEstimateCommand("postgresql", QueryType.DELETE, true, 120L, 90L, - "Seq Scan", 44.5, "{\"operation\":\"Seq Scan\"}", "[raw]", null, false, null, 12); + "Seq Scan", 44.5, null, "{\"operation\":\"Seq Scan\"}", "[raw]", null, false, null, 12); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java index 326c72d26..e0e1cdf37 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java @@ -47,6 +47,7 @@ class DefaultQueryRequestLookupServiceTest { @Mock AiAnalysisRepository aiAnalysisRepository; @Mock ReviewDecisionRepository reviewDecisionRepository; @Mock ApprovalPredictionRepository approvalPredictionRepository; + @Mock com.bablsoft.accessflow.core.internal.persistence.repo.QueryEstimateRepository queryEstimateRepository; @InjectMocks DefaultQueryRequestLookupService service; @Test @@ -333,6 +334,51 @@ void findDetailByIdCopiesFailureFlagAndReason() { assertThat(detail.aiAnalysis().errorMessage()).isEqualTo("provider unavailable"); } + @Test + void findDetailByIdCarriesTheBytesScannedCapAndTheBytesEstimate() { + var orgId = UUID.randomUUID(); + var queryId = UUID.randomUUID(); + var entity = entityWith(queryId, UUID.randomUUID(), orgId, UUID.randomUUID(), + "alice@example.com", QueryStatus.REJECTED); + entity.setBytesScannedCap(1_000L); + entity.setBytesScannedCapSource(com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE); + entity.setBytesScannedCapOutcome(com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + var estimateId = UUID.randomUUID(); + entity.setQueryEstimateId(estimateId); + var estimate = new com.bablsoft.accessflow.core.internal.persistence.entity.QueryEstimateEntity(); + estimate.setId(estimateId); + estimate.setQueryRequest(entity); + estimate.setSupported(true); + estimate.setEstimatedBytesScanned(9_000L); + when(queryEstimateRepository.findById(estimateId)).thenReturn(Optional.of(estimate)); + when(queryRequestRepository.findById(queryId)).thenReturn(Optional.of(entity)); + when(reviewDecisionRepository.findAllByQueryRequest_IdOrderByDecidedAtAsc(queryId)) + .thenReturn(List.of()); + + var detail = service.findDetailById(queryId, orgId).orElseThrow(); + + assertThat(detail.costEstimate().estimatedBytesScanned()).isEqualTo(9_000L); + assertThat(detail.bytesScannedCap().limit()).isEqualTo(1_000L); + assertThat(detail.bytesScannedCap().source()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE); + assertThat(detail.bytesScannedCap().outcome()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + } + + @Test + void findDetailByIdLeavesTheBytesScannedCapNullWhenNoneApplied() { + var orgId = UUID.randomUUID(); + var queryId = UUID.randomUUID(); + var entity = entityWith(queryId, UUID.randomUUID(), orgId, UUID.randomUUID(), + "alice@example.com", QueryStatus.PENDING_AI); + entity.setBytesScannedCap(1_000L); + when(queryRequestRepository.findById(queryId)).thenReturn(Optional.of(entity)); + when(reviewDecisionRepository.findAllByQueryRequest_IdOrderByDecidedAtAsc(queryId)) + .thenReturn(List.of()); + + assertThat(service.findDetailById(queryId, orgId).orElseThrow().bytesScannedCap()).isNull(); + } + @Test void findDetailByIdLeavesAiAnalysisNullWhenNoAnalysisLinked() { var orgId = UUID.randomUUID(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java index f35c1d26e..7df5a0565 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java @@ -131,6 +131,25 @@ void approveByAccessGrantThrowsWhenNotPendingAi() { verify(queryRequestRepository, never()).save(any()); } + @Test + void recordBytesScannedCapStampsTheCapWithoutTransitioning() { + query.setStatus(QueryStatus.PENDING_AI); + when(queryRequestRepository.findByIdForUpdate(queryId)).thenReturn(Optional.of(query)); + + service.recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + + assertThat(query.getStatus()).isEqualTo(QueryStatus.PENDING_AI); + assertThat(query.getBytesScannedCap()).isEqualTo(1_000L); + assertThat(query.getBytesScannedCapSource()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT); + assertThat(query.getBytesScannedCapOutcome()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + verify(queryRequestRepository).save(query); + verify(eventPublisher, never()).publishEvent(any()); + } + @Test void recordApprovalAndAdvancePromotesToApprovedAtLastStage() { query.setStatus(QueryStatus.PENDING_REVIEW); diff --git a/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java index 17e4934b4..0c8bac945 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java @@ -84,7 +84,7 @@ private QueryRequestSnapshot snapshot(QueryType type, boolean transactional) { private QueryEstimateSnapshot persistedSnapshot() { return new QueryEstimateSnapshot(estimateId, queryRequestId, "postgresql", - QueryType.DELETE, true, 100L, 90L, "Seq Scan", 12.5, null, "raw", null, + QueryType.DELETE, true, 100L, 90L, "Seq Scan", 12.5, null, null, "raw", null, false, null, 3, Instant.now()); } @@ -165,6 +165,56 @@ void supportedDryRunWithAffectedCountPersistsFullRow() { .isEqualTo(Duration.ofSeconds(5)); } + @Test + void aWarehouseBytesEstimateIsPersistedWithTheRow() { + when(lookupService.findByQueryRequestId(queryRequestId)) + .thenReturn(Optional.empty()) + .thenReturn(Optional.of(persistedSnapshot())); + when(queryRequestLookupService.findById(queryRequestId)) + .thenReturn(Optional.of(snapshot(QueryType.SELECT, false))); + when(rowSecurityResolutionService.resolveApplicable(any(), any(), any())) + .thenReturn(List.of()); + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.of( + "bigquery", QueryType.SELECT, null, null, null, Set.of(), Duration.ofMillis(4)) + .withEstimatedBytesScanned(3_000_000_000L)); + when(persistenceService.persist(eq(queryRequestId), any())).thenReturn(estimateId); + + service.estimateSubmittedQuery(queryRequestId); + + var captor = ArgumentCaptor.forClass(PersistQueryEstimateCommand.class); + verify(persistenceService).persist(eq(queryRequestId), captor.capture()); + assertThat(captor.getValue().estimatedBytesScanned()).isEqualTo(3_000_000_000L); + } + + @Test + void estimateBytesScannedDryRunsUnderTheEstimateTimeoutAndPersistsNothing() { + var request = new QueryExecutionRequest(datasourceId, "SELECT 1", QueryType.SELECT, 50, + null, List.of(), List.of(), List.of(), false, null, List.of()); + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.of("bigquery", + QueryType.SELECT, null, null, null, Set.of(), Duration.ofMillis(3)) + .withEstimatedBytesScanned(42L)); + + assertThat(service.estimateBytesScanned(request)).contains(42L); + + var captor = ArgumentCaptor.forClass(QueryExecutionRequest.class); + verify(queryExecutor).dryRun(captor.capture()); + assertThat(captor.getValue().statementTimeoutOverride()).isEqualTo(Duration.ofSeconds(5)); + assertThat(captor.getValue().maxRowsOverride()).isEqualTo(50); + verifyNoInteractions(persistenceService, eventPublisher); + } + + @Test + void estimateBytesScannedIsEmptyWhenUnsupportedOrFailing() { + var request = new QueryExecutionRequest(datasourceId, "SELECT 1", QueryType.SELECT, null, + null, List.of(), List.of(), List.of(), false, null, List.of()); + when(queryExecutor.dryRun(any())) + .thenReturn(QueryDryRunResult.unsupported("redis")) + .thenThrow(new IllegalStateException("down")); + + assertThat(service.estimateBytesScanned(request)).isEmpty(); + assertThat(service.estimateBytesScanned(request)).isEmpty(); + } + @Test void writePlanDescendsIntoAccessNodeForScanTypeAndEstimate() { when(lookupService.findByQueryRequestId(queryRequestId)) diff --git a/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java index 7d60404be..d3a70b70b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java @@ -61,6 +61,14 @@ class GroupExecutionServiceTest { private AuditLogService auditLogService; @Mock private org.springframework.context.ApplicationEventPublisher eventPublisher; + @Mock + private com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; + @Mock + private org.springframework.context.MessageSource messageSource; + @Mock + private com.bablsoft.accessflow.proxy.api.QueryCostEstimateService queryCostEstimateService; + @Mock + private com.bablsoft.accessflow.requestgroups.internal.persistence.repo.GroupReviewDecisionRepository decisionRepository; @InjectMocks private GroupExecutionService service; @@ -148,6 +156,121 @@ void ignoresGroupNotApproved() { verify(stateService, org.mockito.Mockito.never()).apply(any(), any()); } + private RequestGroupItemEntity queryItem() { + var item = new RequestGroupItemEntity(); + item.setId(UUID.randomUUID()); + item.setGroupId(group.getId()); + item.setSequenceOrder(0); + item.setTargetKind(RequestGroupTargetKind.QUERY); + item.setDatasourceId(UUID.randomUUID()); + item.setSqlText("SELECT 1"); + item.setQueryType(com.bablsoft.accessflow.core.api.QueryType.SELECT); + when(itemRepository.findByGroupIdOrderBySequenceOrderAsc(group.getId())) + .thenReturn(new ArrayList<>(List.of(item))); + when(permissionLookupService.findFor(any(), any())).thenReturn(java.util.Optional.empty()); + when(maskingPolicyResolutionService.resolveApplicable(any(), any(), any())).thenReturn(List.of()); + when(rowSecurityResolutionService.resolveApplicable(any(), any(), any())).thenReturn(List.of()); + when(datasourceLookupService.findById(any())).thenReturn(java.util.Optional.empty()); + when(queryParser.parse(any(), any())) + .thenReturn(new com.bablsoft.accessflow.core.api.SqlParseResult( + com.bablsoft.accessflow.core.api.QueryType.SELECT, "SELECT 1")); + return item; + } + + private void givenBytesCap(RequestGroupItemEntity item, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + when(bytesScannedCapResolutionService.resolve(item.getDatasourceId(), group.getSubmittedBy())) + .thenReturn(java.util.Optional.of(new com.bablsoft.accessflow.core.api.AppliedBytesCap( + 1_000L, com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + missing))); + } + + @Test + void aQueryMemberOverTheBytesCapFailsWithoutRunning() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.of(5_000L)); + when(messageSource.getMessage(org.mockito.ArgumentMatchers.eq("error.bytes_cap.exceeded"), + any(), any())).thenReturn("over the cap"); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.FAILED); + assertThat(item.getErrorMessage()).isEqualTo("over the cap"); + verify(queryExecutor, org.mockito.Mockito.never()).execute(any()); + var audit = org.mockito.ArgumentCaptor.forClass( + com.bablsoft.accessflow.audit.api.AuditEntry.class); + verify(auditLogService, org.mockito.Mockito.atLeastOnce()).record(audit.capture()); + assertThat(audit.getAllValues()).anySatisfy(entry -> { + assertThat(entry.action()) + .isEqualTo(com.bablsoft.accessflow.audit.api.AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED); + assertThat(entry.metadata()).containsEntry("estimated_bytes", 5_000L) + .containsEntry("item_id", item.getId().toString()); + }); + } + + @Test + void aQueryMemberWithoutAnEstimateFailsUnderReject() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.empty()); + when(messageSource.getMessage(org.mockito.ArgumentMatchers.eq("error.bytes_cap.no_estimate"), + any(), any())).thenReturn("no estimate"); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.FAILED); + assertThat(item.getErrorMessage()).isEqualTo("no estimate"); + } + + @Test + void aQueryMemberWithoutAnEstimateRunsUnderRequireReviewOnceAPersonApproved() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.empty()); + when(decisionRepository.existsByRequestGroupIdAndDecision(group.getId(), + com.bablsoft.accessflow.core.api.DecisionType.APPROVED)).thenReturn(true); + when(queryExecutor.execute(any())).thenReturn( + new com.bablsoft.accessflow.core.api.SelectExecutionResult( + List.of(), List.of(), 1L, false, java.time.Duration.ofMillis(3))); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.EXECUTED); + } + + @Test + void aQueryMemberWithoutAnEstimateIsRefusedWhenNoPersonApprovedTheGroup() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.empty()); + when(messageSource.getMessage( + org.mockito.ArgumentMatchers.eq("error.bytes_cap.no_estimate_unreviewed"), any(), + any())).thenReturn("never reviewed"); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.FAILED); + assertThat(item.getErrorMessage()).isEqualTo("never reviewed"); + verify(queryExecutor, org.mockito.Mockito.never()).execute(any()); + } + + @Test + void anUncappedQueryMemberIsNeverDryRun() { + queryItem(); + when(queryExecutor.execute(any())).thenReturn( + new com.bablsoft.accessflow.core.api.SelectExecutionResult( + List.of(), List.of(), 1L, false, java.time.Duration.ofMillis(3))); + + service.execute(group.getId(), null, "manual"); + + verify(queryCostEstimateService, org.mockito.Mockito.never()).estimateBytesScanned(any()); + } + @Test void runsQueryMemberThroughProxyExecutor() { var item = new RequestGroupItemEntity(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java index 0343cf896..c0e3389cf 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java @@ -520,6 +520,101 @@ void updateDatasourceSetsKeepsAndClearsTheEnvironment() { assertThat(datasourceRepository.findById(ds.getId()).orElseThrow().getEnvironment()).isNull(); } + @Test + void updateDatasourceSetsKeepsAndClearsTheBytesScannedCapOnAWarehouse() { + var ds = saveDatasource(primaryOrg, "WH"); + ds.setDbType(com.bablsoft.accessflow.core.api.DbType.BIGQUERY); + datasourceRepository.save(ds); + + var set = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"max_bytes_scanned_per_query":1000000000000,"bytes_cap_missing_estimate":"REJECT"} + """) + .exchange(); + assertThat(set).hasStatus(200); + assertThat(set).bodyJson().extractingPath("$.max_bytes_scanned_per_query").asNumber() + .isEqualTo(1_000_000_000_000L); + assertThat(set).bodyJson().extractingPath("$.bytes_cap_missing_estimate").asString() + .isEqualTo("REJECT"); + + var keep = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"host":"other-host"} + """) + .exchange(); + assertThat(keep).bodyJson().extractingPath("$.max_bytes_scanned_per_query").asNumber() + .isEqualTo(1_000_000_000_000L); + + var clear = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"clear_max_bytes_scanned_per_query":true} + """) + .exchange(); + assertThat(clear).hasStatus(200); + assertThat(clear).bodyJson().doesNotHavePath("$.max_bytes_scanned_per_query"); + } + + @Test + void aBytesScannedCapIsRefusedOnAnEngineWithoutABytesEstimateAndBelowOne() { + var ds = saveDatasource(primaryOrg, "DS"); + + var unsupported = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"max_bytes_scanned_per_query":5} + """) + .exchange(); + assertThat(unsupported).hasStatus(422); + assertThat(unsupported).bodyJson().extractingPath("$.error").asString() + .isEqualTo("BYTES_SCANNED_CAP_NOT_SUPPORTED"); + + var zero = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"max_bytes_scanned_per_query":0} + """) + .exchange(); + assertThat(zero).hasStatus(400); + + var grant = mvc.post().uri("/api/v1/datasources/" + ds.getId() + "/permissions") + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"user_id":"%s","can_read":true,"bytes_scanned_limit_override":5} + """.formatted(analyst.getId())) + .exchange(); + assertThat(grant).hasStatus(422); + assertThat(permissionRepository.existsByUser_IdAndDatasource_Id(analyst.getId(), + ds.getId())).isFalse(); + } + + @Test + void grantPermissionRoundTripsTheBytesScannedOverrideOnAWarehouse() { + var ds = saveDatasource(primaryOrg, "WH"); + ds.setDbType(com.bablsoft.accessflow.core.api.DbType.SNOWFLAKE); + datasourceRepository.save(ds); + + var result = mvc.post().uri("/api/v1/datasources/" + ds.getId() + "/permissions") + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"user_id":"%s","can_read":true,"bytes_scanned_limit_override":500000000000} + """.formatted(analyst.getId())) + .exchange(); + + assertThat(result).hasStatus(201); + assertThat(result).bodyJson().extractingPath("$.bytes_scanned_limit_override").asNumber() + .isEqualTo(500_000_000_000L); + } + @Test void updateDatasourcePasswordReencryptsIt() { var ds = saveDatasource(primaryOrg, "DS"); diff --git a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java index 156aa50e5..0f6fd1cf0 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java @@ -303,6 +303,18 @@ void deniedShapesNotSupportedReturns422() { .containsEntry("dbType", "REDIS"); } + @Test + void bytesScannedCapNotSupportedReturns422() { + var pd = handler.handleBytesScannedCapNotSupported( + new com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException( + com.bablsoft.accessflow.core.api.DbType.POSTGRESQL)); + + assertThat(pd.getStatus()).isEqualTo(422); + assertThat(pd.getProperties()) + .containsEntry("error", "BYTES_SCANNED_CAP_NOT_SUPPORTED") + .containsEntry("dbType", "POSTGRESQL"); + } + @Test void invalidSqlReturns422() { var pd = handler.handleInvalidSql(new InvalidSqlException("nope")); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java index 6b32b3085..f0b917889 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java @@ -169,4 +169,24 @@ void ciCdOriginExposesExpected() { assertThat(new ConditionNode.CiCdOrigin(true).expected()).isTrue(); assertThat(new ConditionNode.CiCdOrigin(false).expected()).isFalse(); } + + @Test + void estimatedBytesScannedRejectsANullOperatorAndANegativeValue() { + assertThatThrownBy(() -> new ConditionNode.EstimatedBytesScanned(null, 5)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, -1)) + .isInstanceOf(IllegalArgumentException.class); + var node = new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, 1_000_000L); + assertThat(node.operator()).isEqualTo(ComparisonOperator.GT); + assertThat(node.value()).isEqualTo(1_000_000L); + } + + @Test + void theShapeCompatibleContextConstructorLeavesTheBytesEstimateAbsent() { + var context = new ConditionContext(QueryType.SELECT, Set.of(), RiskLevel.LOW, 1, "ANALYST", + Set.of(), java.time.LocalDateTime.now(), false, false, false, null, null, false, + null, false, 10L, "Seq Scan", Set.of(QueryShape.JOIN), true); + assertThat(context.estimatedBytesScanned()).isNull(); + assertThat(context.shapesAnalyzed()).isTrue(); + } } diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java new file mode 100644 index 000000000..ed8f84ba2 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java @@ -0,0 +1,48 @@ +package com.bablsoft.accessflow.workflow.internal; + +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class BytesCapCheckTest { + + private final AppliedBytesCap cap = new AppliedBytesCap(100, BytesScannedCapSource.DATASOURCE, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + @Test + void ofComparesTheEstimate() { + var over = BytesCapCheck.of(cap, 101L); + assertThat(over.outcome()).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + assertThat(over.rejects()).isTrue(); + assertThat(over.forcesReview()).isFalse(); + assertThat(over.limit()).isEqualTo(100); + assertThat(over.estimatedBytes()).isEqualTo(101L); + } + + @Test + void aMissingEstimateUnderRequireReviewForcesReviewWithoutRejecting() { + var check = BytesCapCheck.of(cap, null); + assertThat(check.forcesReview()).isTrue(); + assertThat(check.rejects()).isFalse(); + } + + @Test + void anUnevaluatedCapNeitherRejectsNorForcesReview() { + var check = BytesCapCheck.unevaluated(cap); + assertThat(check.outcome()).isNull(); + assertThat(check.rejects()).isFalse(); + assertThat(check.forcesReview()).isFalse(); + assertThat(check.source()).isEqualTo(BytesScannedCapSource.DATASOURCE); + } + + @Test + void aSourceIsRequired() { + assertThatThrownBy(() -> new BytesCapCheck(1, null, null, null)) + .isInstanceOf(NullPointerException.class); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java new file mode 100644 index 000000000..165db3237 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java @@ -0,0 +1,355 @@ +package com.bablsoft.accessflow.workflow.internal; + +import com.bablsoft.accessflow.TestcontainersConfig; +import com.bablsoft.accessflow.core.api.AuthProviderType; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import com.bablsoft.accessflow.core.api.CredentialEncryptionService; +import com.bablsoft.accessflow.core.api.DbType; +import com.bablsoft.accessflow.core.api.QueryDryRunResult; +import com.bablsoft.accessflow.core.api.QueryStatus; +import com.bablsoft.accessflow.core.api.QueryType; +import com.bablsoft.accessflow.core.api.RiskLevel; +import com.bablsoft.accessflow.core.api.SelectExecutionResult; +import com.bablsoft.accessflow.core.api.SslMode; +import com.bablsoft.accessflow.core.api.UserRoleType; +import com.bablsoft.accessflow.core.events.AiAnalysisCompletedEvent; +import com.bablsoft.accessflow.core.events.AiAnalysisSkippedEvent; +import com.bablsoft.accessflow.core.internal.persistence.entity.DatasourceEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.OrganizationEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.QueryRequestEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.ReviewPlanApproverEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.ReviewPlanEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.UserEntity; +import com.bablsoft.accessflow.core.internal.persistence.repo.DatasourceRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.OrganizationRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.QueryEstimateRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.QueryRequestRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.ReviewPlanApproverRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.ReviewPlanRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.UserRepository; +import com.bablsoft.accessflow.core.api.SqlParseResult; +import com.bablsoft.accessflow.proxy.api.QueryExecutor; +import com.bablsoft.accessflow.proxy.api.QueryParser; +import com.bablsoft.accessflow.workflow.api.ComparisonOperator; +import com.bablsoft.accessflow.workflow.api.ConditionNode; +import com.bablsoft.accessflow.workflow.api.QueryLifecycleService; +import com.bablsoft.accessflow.workflow.api.QueryLifecycleService.ExecuteQueryCommand; +import com.bablsoft.accessflow.workflow.api.RoutingAction; +import com.bablsoft.accessflow.workflow.internal.persistence.entity.RoutingPolicyEntity; +import com.bablsoft.accessflow.workflow.internal.persistence.repo.RoutingDecisionRepository; +import com.bablsoft.accessflow.workflow.internal.persistence.repo.RoutingPolicyRepository; +import com.bablsoft.accessflow.workflow.internal.routing.RoutingConditionCodec; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.testcontainers.context.ImportTestcontainers; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * #941 end to end: the warehouse's pre-flight bytes estimate is persisted with the query's + * estimate, a bytes-scanned cap refuses the query when it leaves {@code PENDING_AI}, a missing + * estimate under {@code REQUIRE_REVIEW} holds an automatic approval for a person, the cap is + * re-checked just before execution, and an {@code estimated_bytes_scanned} routing policy sees the + * estimate on the AI-skipped path, where nothing else waits for it. + * + *

No warehouse is contacted: {@link QueryExecutor} is mocked, so its {@code dryRun} supplies the + * estimate the BigQuery engine would report. + */ +@SpringBootTest +@ImportTestcontainers(TestcontainersConfig.class) +class BytesScannedCapEnforcementIntegrationTest { + + private static final String SQL = "SELECT * FROM events"; + private static final long ONE_TB = 1_000_000_000_000L; + + @MockitoBean QueryExecutor queryExecutor; + // No BigQuery engine plugin is loaded in tests; the parser is the plugin's to supply. + @MockitoBean QueryParser queryParser; + + @Autowired ApplicationEventPublisher eventPublisher; + @Autowired PlatformTransactionManager transactionManager; + @Autowired QueryLifecycleService queryLifecycleService; + @Autowired OrganizationRepository organizationRepository; + @Autowired UserRepository userRepository; + @Autowired DatasourceRepository datasourceRepository; + @Autowired QueryRequestRepository queryRequestRepository; + @Autowired QueryEstimateRepository queryEstimateRepository; + @Autowired ReviewPlanRepository reviewPlanRepository; + @Autowired ReviewPlanApproverRepository reviewPlanApproverRepository; + @Autowired RoutingPolicyRepository routingPolicyRepository; + @Autowired RoutingDecisionRepository routingDecisionRepository; + @Autowired RoutingConditionCodec routingConditionCodec; + @Autowired CredentialEncryptionService encryptionService; + @Autowired JdbcTemplate jdbcTemplate; + + private OrganizationEntity organization; + private UserEntity submitter; + + @BeforeEach + void setUp() { + cleanup(); + organization = new OrganizationEntity(); + organization.setId(UUID.randomUUID()); + organization.setName("Primary"); + organization.setSlug("primary-" + UUID.randomUUID()); + organizationRepository.save(organization); + submitter = persistUser(); + when(queryParser.parse(any(), any())).thenReturn(new SqlParseResult(QueryType.SELECT, SQL)); + } + + @AfterEach + void cleanup() { + if (organization != null) { + jdbcTemplate.update("DELETE FROM audit_log WHERE organization_id = ?", organization.getId()); + } + routingDecisionRepository.deleteAll(); + routingPolicyRepository.deleteAll(); + jdbcTemplate.update("UPDATE query_requests SET query_estimate_id = NULL"); + queryEstimateRepository.deleteAll(); + queryRequestRepository.deleteAll(); + datasourceRepository.deleteAll(); + reviewPlanApproverRepository.deleteAll(); + reviewPlanRepository.deleteAll(); + userRepository.deleteAll(); + organizationRepository.deleteAll(); + } + + @Test + void anEstimateOverTheCapIsRejectedWhenTheQueryLeavesPendingAi() { + givenBytesEstimate(2 * ONE_TB); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisCompletedEvent(query.getId(), null, RiskLevel.LOW, 5)); + + awaitStatus(query.getId(), QueryStatus.REJECTED); + var row = queryRequestRepository.findById(query.getId()).orElseThrow(); + assertThat(row.getBytesScannedCap()).isEqualTo(ONE_TB); + assertThat(row.getBytesScannedCapSource()).isEqualTo(BytesScannedCapSource.DATASOURCE); + assertThat(row.getBytesScannedCapOutcome()).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + assertThat(queryEstimateRepository.findByQueryRequestId(query.getId()).orElseThrow() + .getEstimatedBytesScanned()).isEqualTo(2 * ONE_TB); + assertThat(capAuditMetadata(query.getId())) + .contains("\"stage\": \"decision\"") + .contains("\"outcome\": \"EXCEEDED\""); + } + + @Test + void aMissingEstimateUnderRequireReviewHoldsAnAutoApprovalForAPerson() { + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.unsupported("bigquery")); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisSkippedEvent(query.getId(), "ai_analysis_enabled=false")); + + awaitStatus(query.getId(), QueryStatus.PENDING_REVIEW); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow() + .getBytesScannedCapOutcome()).isEqualTo(BytesScannedCapOutcome.NO_ESTIMATE_REVIEW); + assertThat(capAuditMetadata(query.getId())).contains("NO_ESTIMATE_REVIEW"); + } + + @Test + void aCapLoweredAfterApprovalFailsTheExecutionBeforeTheWarehouseRunsIt() { + givenBytesEstimate(500_000_000_000L); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var query = persistPendingAiQuery(datasource); + publish(new AiAnalysisCompletedEvent(query.getId(), null, RiskLevel.LOW, 5)); + awaitStatus(query.getId(), QueryStatus.APPROVED); + + datasource.setMaxBytesScannedPerQuery(100_000_000_000L); + datasourceRepository.save(datasource); + var outcome = queryLifecycleService.execute(new ExecuteQueryCommand(query.getId(), + submitter.getId(), organization.getId(), false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.FAILED); + verify(queryParser, never()).parse(any(), any()); + verify(queryExecutor, never()).execute(any()); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow().getErrorMessage()) + .contains("500 GB").contains("100 GB"); + assertThat(capAuditMetadata(query.getId())).contains("\"stage\": \"execution\""); + } + + @Test + void anEstimateWithinTheCapExecutes() { + givenBytesEstimate(10L); + when(queryExecutor.execute(any())).thenReturn(new SelectExecutionResult(List.of(), + List.of(), 0L, false, Duration.ofMillis(3))); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REJECT); + var query = persistPendingAiQuery(datasource); + publish(new AiAnalysisCompletedEvent(query.getId(), null, RiskLevel.LOW, 5)); + awaitStatus(query.getId(), QueryStatus.APPROVED); + + var outcome = queryLifecycleService.execute(new ExecuteQueryCommand(query.getId(), + submitter.getId(), organization.getId(), false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.EXECUTED); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow() + .getBytesScannedCapOutcome()).isEqualTo(BytesScannedCapOutcome.WITHIN); + assertThat(jdbcTemplate.queryForObject("SELECT count(*) FROM audit_log WHERE resource_id = ? " + + "AND action = 'QUERY_BYTES_SCANNED_CAP_ENFORCED'", Integer.class, query.getId())) + .isZero(); + } + + @Test + void anEstimatedBytesPolicyEscalatesOnTheSkippedPathWithoutAnyCap() { + givenBytesEstimate(2 * ONE_TB); + var datasource = persistDatasource(persistPlan(false), null, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var policy = persistPolicy(new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, + ONE_TB)); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisSkippedEvent(query.getId(), "ai_analysis_enabled=false")); + + awaitStatus(query.getId(), QueryStatus.PENDING_REVIEW); + assertThat(routingDecisionRepository.findByQueryRequestId(query.getId()).orElseThrow() + .getMatchedPolicyId()).isEqualTo(policy.getId()); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow() + .getBytesScannedCap()).isNull(); + } + + @Test + void anEstimatedBytesPolicyDoesNotFireWithoutAnEstimate() { + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.unsupported("bigquery")); + var datasource = persistDatasource(persistPlan(false), null, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + persistPolicy(new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GTE, 0)); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisSkippedEvent(query.getId(), "ai_analysis_enabled=false")); + + awaitStatus(query.getId(), QueryStatus.APPROVED); + assertThat(routingDecisionRepository.findByQueryRequestId(query.getId())).isEmpty(); + } + + // ── Fixtures ────────────────────────────────────────────────────────────── + + private void givenBytesEstimate(long bytes) { + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.of("bigquery", + QueryType.SELECT, null, null, null, Set.of(), Duration.ofMillis(2)) + .withEstimatedBytesScanned(bytes)); + } + + private void publish(Object event) { + new TransactionTemplate(transactionManager) + .executeWithoutResult(status -> eventPublisher.publishEvent(event)); + } + + private void awaitStatus(UUID queryId, QueryStatus expected) { + Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> + assertThat(queryRequestRepository.findById(queryId).orElseThrow().getStatus()) + .isEqualTo(expected)); + } + + private String capAuditMetadata(UUID queryId) { + var rows = jdbcTemplate.queryForList("SELECT metadata::text FROM audit_log WHERE " + + "resource_id = ? AND action = 'QUERY_BYTES_SCANNED_CAP_ENFORCED'", String.class, + queryId); + assertThat(rows).hasSize(1); + return rows.get(0); + } + + private UserEntity persistUser() { + var user = new UserEntity(); + user.setId(UUID.randomUUID()); + user.setEmail("submitter-" + UUID.randomUUID() + "@example.com"); + user.setDisplayName("submitter"); + user.setPasswordHash("hash"); + user.setRole(UserRoleType.ANALYST); + user.setAuthProvider(AuthProviderType.LOCAL); + user.setActive(true); + user.setOrganization(organization); + return userRepository.save(user); + } + + private ReviewPlanEntity persistPlan(boolean requiresHumanApproval) { + var plan = new ReviewPlanEntity(); + plan.setId(UUID.randomUUID()); + plan.setOrganization(organization); + plan.setName("plan-" + UUID.randomUUID()); + plan.setRequiresAiReview(false); + plan.setRequiresHumanApproval(requiresHumanApproval); + plan.setMinApprovalsRequired(1); + plan.setApprovalTimeoutHours(24); + plan.setAutoApproveReads(false); + reviewPlanRepository.save(plan); + var rule = new ReviewPlanApproverEntity(); + rule.setId(UUID.randomUUID()); + rule.setReviewPlan(plan); + rule.setRole("REVIEWER"); + rule.setStage(1); + reviewPlanApproverRepository.save(rule); + return plan; + } + + private DatasourceEntity persistDatasource(ReviewPlanEntity plan, Long cap, + BytesCapMissingEstimateAction missing) { + var ds = new DatasourceEntity(); + ds.setId(UUID.randomUUID()); + ds.setOrganization(organization); + ds.setName("WH-" + UUID.randomUUID()); + ds.setDbType(DbType.BIGQUERY); + ds.setHost("bigquery.invalid"); + ds.setPort(443); + ds.setDatabaseName("analytics"); + ds.setUsername("svc"); + ds.setPasswordEncrypted(encryptionService.encrypt("{}")); + ds.setSslMode(SslMode.REQUIRE); + ds.setConnectionPoolSize(5); + ds.setMaxRowsPerQuery(1000); + ds.setReviewPlan(plan); + ds.setAiAnalysisEnabled(false); + ds.setActive(true); + ds.setMaxBytesScannedPerQuery(cap); + ds.setBytesCapMissingEstimate(missing); + return datasourceRepository.save(ds); + } + + private QueryRequestEntity persistPendingAiQuery(DatasourceEntity ds) { + var query = new QueryRequestEntity(); + query.setId(UUID.randomUUID()); + query.setDatasource(ds); + query.setSubmittedBy(submitter); + query.setSqlText(SQL); + query.setQueryType(QueryType.SELECT); + query.setStatus(QueryStatus.PENDING_AI); + return queryRequestRepository.save(query); + } + + private RoutingPolicyEntity persistPolicy(ConditionNode condition) { + var policy = new RoutingPolicyEntity(); + policy.setId(UUID.randomUUID()); + policy.setOrganizationId(organization.getId()); + policy.setName("policy-" + UUID.randomUUID()); + policy.setPriority(1); + policy.setEnabled(true); + policy.setAction(RoutingAction.ESCALATE); + policy.setRequiredApprovals(1); + policy.setConditionJson(routingConditionCodec.encode(condition)); + return routingPolicyRepository.save(policy); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java index 0afae9775..99719f337 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java @@ -98,6 +98,7 @@ class DefaultAccessSimulationServiceTest { @Mock RowSecurityClassificationService rowSecurityClassificationService; @Mock MaskingPolicyResolutionService maskingPolicyResolutionService; @Mock BreakGlassEligibilityService breakGlassEligibilityService; + @Mock com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; private DefaultAccessSimulationService service; @@ -114,7 +115,7 @@ void buildService() { routingPolicyEngine, reviewPlanLookupService, reviewerEligibilityService, rowSecurityResolutionService, rowSecurityClassificationService, maskingPolicyResolutionService, - breakGlassEligibilityService); + breakGlassEligibilityService, bytesScannedCapResolutionService); service.setClock(clock); } @@ -135,7 +136,7 @@ void stubHappyPath() { when(permissionLookupService.findFor(userId, datasourceId)) .thenReturn(Optional.of(permission(true, false, false, List.of("public")))); when(queryDecisionEvaluator.evaluate(any(), any(), any(), org.mockito.ArgumentMatchers.anyInt(), - any(), any())).thenReturn(planDecision(QueryStatus.PENDING_REVIEW)); + any(), any(), any())).thenReturn(planDecision(QueryStatus.PENDING_REVIEW)); when(sqlReviewService.evaluate(eq(organizationId), eq(datasourceId), any())) .thenReturn(SqlReviewResult.clean()); when(routingPolicyEngine.enabledFor(organizationId, datasourceId)).thenReturn(List.of()); @@ -175,7 +176,7 @@ void aSimulationNeverAsksTheEvaluatorToApplyAnything() { service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); verify(queryDecisionEvaluator).evaluate(any(), eq(AiOutcome.COMPLETED), eq(RiskLevel.LOW), - eq(5), eq(List.of()), eq(clock)); + eq(5), eq(List.of()), org.mockito.ArgumentMatchers.isNull(), eq(clock)); verify(datasourceAdminService, never()).update(any(), any(), any()); } @@ -195,7 +196,8 @@ void theSimulatorHandsTheEvaluatorTheSameBlockingRuleIdsTheLivePathWouldRead() { service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); verify(queryDecisionEvaluator).evaluate(any(), eq(AiOutcome.COMPLETED), eq(RiskLevel.LOW), - eq(5), eq(List.of("cross_join", "select_star")), eq(clock)); + eq(5), eq(List.of("cross_join", "select_star")), + org.mockito.ArgumentMatchers.isNull(), eq(clock)); verify(sqlReviewService).evaluate(organizationId, datasourceId, "SELECT card_number FROM public.payments"); } @@ -208,7 +210,24 @@ void aNonRelationalDatasourceContributesNoBlockingRuleIds() { service.simulate(organizationId, input(AiOutcome.SKIPPED, null, null)); verify(queryDecisionEvaluator).evaluate(any(), eq(AiOutcome.SKIPPED), eq(null), eq(-1), - eq(List.of()), eq(clock)); + eq(List.of()), org.mockito.ArgumentMatchers.isNull(), eq(clock)); + } + + @Test + void aBytesCapIsHandedToTheEvaluatorUnevaluatedBecauseASimulationHasNoEstimate() { + when(bytesScannedCapResolutionService.resolve(datasourceId, userId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.AppliedBytesCap(500L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT))); + + service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); + + var passed = org.mockito.ArgumentCaptor.forClass(BytesCapCheck.class); + verify(queryDecisionEvaluator).evaluate(any(), any(), any(), + org.mockito.ArgumentMatchers.anyInt(), any(), passed.capture(), any()); + assertThat(passed.getValue().limit()).isEqualTo(500L); + assertThat(passed.getValue().outcome()).isNull(); + assertThat(passed.getValue().rejects()).isFalse(); } // ── Shape ───────────────────────────────────────────────────────────────── @@ -236,7 +255,7 @@ void everyStageIsStillReportedWhenTheRequestIsBlockedEarly() { assertThat(step(result.steps(), QueryDecisionStepKind.ROUTING_POLICIES).outcome()) .isEqualTo(StepOutcome.SKIP); verify(queryDecisionEvaluator, never()).evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), any()); + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any()); } @Test @@ -514,7 +533,7 @@ void theRoutingStepCarriesEveryPolicyNotJustTheWinner() { @Test void reviewersAreSkippedWhenTheRequestWouldNotReachReview() { when(queryDecisionEvaluator.evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), any())) + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any())) .thenReturn(planDecision(QueryStatus.APPROVED)); var result = service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); @@ -534,13 +553,13 @@ void theSimulatorEvaluatesInTheSameZoneTheLiveListenerDoes() { routingPolicyEngine, reviewPlanLookupService, reviewerEligibilityService, rowSecurityResolutionService, rowSecurityClassificationService, maskingPolicyResolutionService, - breakGlassEligibilityService); + breakGlassEligibilityService, bytesScannedCapResolutionService); fresh.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); var passed = org.mockito.ArgumentCaptor.forClass(Clock.class); verify(queryDecisionEvaluator).evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), passed.capture()); + org.mockito.ArgumentMatchers.anyInt(), any(), any(), passed.capture()); assertThat(passed.getValue().getZone()).isEqualTo(ZoneId.systemDefault()); } @@ -583,7 +602,7 @@ void theDecisivePolicyIsTheOneTheDecisionNamesNotASecondEvaluationsOwnFirstMatch when(routingPolicyEngine.evaluateAll(any(), any())) .thenReturn(List.of(alsoMatched, decided)); when(queryDecisionEvaluator.evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), any())) + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any())) .thenReturn(routedDecision(decidedId)); var result = service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); @@ -724,6 +743,7 @@ private AccessSimulationInput input(AiOutcome outcome, RiskLevel level, Integer private QueryDecision planDecision(QueryStatus status) { var steps = List.of( DecisionTraceStep.of(QueryDecisionStepKind.SQL_REVIEW, StepOutcome.NO_MATCH, "k"), + DecisionTraceStep.of(QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.DENY, "k")); @@ -743,6 +763,7 @@ private QueryDecision routedDecision(UUID policyId) { "P", com.bablsoft.accessflow.workflow.api.RoutingAction.ESCALATE, 1, "matched"); var steps = List.of( DecisionTraceStep.of(QueryDecisionStepKind.SQL_REVIEW, StepOutcome.NO_MATCH, "k"), + DecisionTraceStep.of(QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.SKIP, "k"), DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.SKIP, "k")); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java index 8ffa5b9a9..19d52f703 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java @@ -95,6 +95,8 @@ class DefaultQueryLifecycleServiceTest { @Mock AuditLogService auditLogService; @Mock MessageSource messageSource; @Mock ApplicationEventPublisher eventPublisher; + @Mock com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; + @Mock com.bablsoft.accessflow.proxy.api.QueryCostEstimateService queryCostEstimateService; DefaultQueryLifecycleService service; @@ -132,7 +134,9 @@ void setUp() { auditLogService, new ObjectMapper(), messageSource, - eventPublisher); + eventPublisher, + bytesScannedCapResolutionService, + queryCostEstimateService); when(queryParser.parse(anyString(), any())).thenAnswer(inv -> { String sql = inv.getArgument(0); return new SqlParseResult(QueryType.SELECT, sql); @@ -618,6 +622,108 @@ void executeThrowsNotFoundWhenQueryMissing() { // ── execute (failure) ───────────────────────────────────────────────────── + // ── execute: bytes-scanned cap (#941) ───────────────────────────────────── + + private void givenBytesCap(long limit, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + when(bytesScannedCapResolutionService.resolve(datasourceId, submitterId)) + .thenReturn(Optional.of(new com.bablsoft.accessflow.core.api.AppliedBytesCap(limit, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, missing))); + } + + private void givenBytesEstimate(Long bytes) { + when(queryCostEstimateService.estimateSubmittedQuery(queryId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), + queryId, "bigquery", QueryType.SELECT, true, null, null, null, null, + bytes, null, null, null, false, null, 5, java.time.Instant.now()))); + } + + @Test + void executeFailsBeforeTheExecutorWhenTheEstimateExceedsTheCap() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenBytesEstimate(9_000L); + when(messageSource.getMessage(eq("error.bytes_cap.exceeded"), any(), any())) + .thenReturn("estimate over the cap"); + + var outcome = service.execute(new ExecuteQueryCommand(queryId, submitterId, + organizationId, false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.FAILED); + verify(queryExecutor, never()).execute(any()); + var execCaptor = ArgumentCaptor.forClass(RecordExecutionCommand.class); + verify(queryRequestStateService).recordExecutionOutcome(execCaptor.capture()); + assertThat(execCaptor.getValue().errorMessage()).isEqualTo("estimate over the cap"); + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService, org.mockito.Mockito.times(2)).record(auditCaptor.capture()); + assertThat(auditCaptor.getAllValues()).extracting(AuditEntry::action).containsExactly( + AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, AuditAction.QUERY_FAILED); + assertThat(auditCaptor.getAllValues().get(0).metadata()) + .containsEntry("stage", "execution") + .containsEntry("estimated_bytes", 9_000L) + .containsEntry("source", "GRANT"); + } + + @Test + void executeFailsAMissingEstimateUnderReject() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + when(queryCostEstimateService.estimateSubmittedQuery(queryId)) + .thenThrow(new IllegalStateException("warehouse down")); + when(messageSource.getMessage(eq("error.bytes_cap.no_estimate"), any(), any())) + .thenReturn("no estimate"); + + var outcome = service.executeBreakGlass(queryId, submitterId); + + assertThat(outcome.status()).isEqualTo(QueryStatus.FAILED); + verify(queryExecutor, never()).execute(any()); + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService, org.mockito.Mockito.times(2)).record(auditCaptor.capture()); + assertThat(auditCaptor.getAllValues().get(0).metadata()) + .doesNotContainKey("estimated_bytes") + .containsEntry("outcome", "NO_ESTIMATE_REJECTED"); + } + + @Test + void executeRunsAMissingEstimateUnderRequireReviewAndRecordsTheCap() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryExecutor.execute(any())).thenReturn(new SelectExecutionResult( + List.of(new ResultColumn("id", 4, "int4")), List.of(List.of(1)), 1L, false, + Duration.ofMillis(5))); + + var outcome = service.execute(new ExecuteQueryCommand(queryId, submitterId, + organizationId, false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.EXECUTED); + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(auditCaptor.capture()); + assertThat(auditCaptor.getValue().metadata()) + .containsEntry("bytes_scanned_cap", 1_000L) + .containsEntry("bytes_scanned_cap_source", "GRANT") + .doesNotContainKey("bytes_scanned_estimate"); + } + + @Test + void executeWithinTheCapRecordsTheEstimateOnTheExecutedRow() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + givenBytesEstimate(10L); + when(queryExecutor.execute(any())).thenReturn(new SelectExecutionResult( + List.of(new ResultColumn("id", 4, "int4")), List.of(List.of(1)), 1L, false, + Duration.ofMillis(5))); + + service.execute(new ExecuteQueryCommand(queryId, submitterId, organizationId, false)); + + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(auditCaptor.capture()); + assertThat(auditCaptor.getValue().metadata()).containsEntry("bytes_scanned_estimate", 10L); + } + @Test void executeRecordsFailureWhenExecutorThrows() { when(queryRequestLookupService.findById(queryId)) diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java index 243da3060..5945435eb 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java @@ -5,6 +5,8 @@ import com.bablsoft.accessflow.access.api.AccessGrantView; import com.bablsoft.accessflow.ai.api.BehaviorAnomalyLookupService; import com.bablsoft.accessflow.core.api.ApproverRule; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.QueryEstimateLookupService; import com.bablsoft.accessflow.core.api.QueryRequestLookupService; import com.bablsoft.accessflow.core.api.QueryRequestSnapshot; @@ -112,6 +114,8 @@ void aiFailureTraceSkipsEveryDecisionStage() { assertThat(trace.steps()).extracting("step", "outcome").containsExactly( org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.SQL_REVIEW, StepOutcome.NO_MATCH), + org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.NO_MATCH), org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.SKIP), org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.GRANT_FAST_PATH, @@ -430,8 +434,9 @@ void thePlanFallThroughTraceCoversAllThreeStages() { 5, List.of(), clock).trace(); assertThat(trace.steps()).extracting("step").containsExactly( - QueryDecisionStepKind.SQL_REVIEW, QueryDecisionStepKind.ROUTING_POLICIES, - QueryDecisionStepKind.GRANT_FAST_PATH, QueryDecisionStepKind.REVIEW_PLAN); + QueryDecisionStepKind.SQL_REVIEW, QueryDecisionStepKind.BYTES_SCANNED_CAP, + QueryDecisionStepKind.ROUTING_POLICIES, QueryDecisionStepKind.GRANT_FAST_PATH, + QueryDecisionStepKind.REVIEW_PLAN); assertThat(step(trace, QueryDecisionStepKind.ROUTING_POLICIES).outcome()) .isEqualTo(StepOutcome.NO_MATCH); assertThat(step(trace, QueryDecisionStepKind.REVIEW_PLAN).outcome()).isEqualTo(StepOutcome.DENY); @@ -630,6 +635,159 @@ void theAiFailedPathStillRecordsTheBlockOnTheTrace() { assertThat(sqlReview.details()).containsEntry("blocking_rule_ids", List.of("select_star")); } + // ── Bytes-scanned cap (#941) ────────────────────────────────────────────── + + private static BytesCapCheck cap(Long estimated, BytesScannedCapOutcome outcome) { + return new BytesCapCheck(1_000_000_000_000L, BytesScannedCapSource.DATASOURCE, estimated, + outcome); + } + + @Test + void anExceededCapRejectsBeforeRoutingIsConsulted() { + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), + cap(2_000_000_000_000L, BytesScannedCapOutcome.EXCEEDED), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.BYTES_CAP_REJECTED); + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.REJECTED); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + assertThat(decision.context()).isNull(); + var capStep = step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP); + assertThat(capStep.outcome()).isEqualTo(StepOutcome.DENY); + assertThat(capStep.reasonKey()).isEqualTo("workflow.decision.bytes_cap.exceeded"); + assertThat(capStep.reasonArgs()).containsExactly( + "2 TB (2000000000000 B)", "1 TB (1000000000000 B)"); + assertThat(capStep.details()).containsEntry("bytes_scanned_cap_source", "DATASOURCE"); + assertThat(step(decision.trace(), QueryDecisionStepKind.ROUTING_POLICIES).reasonKey()) + .isEqualTo("workflow.decision.routing.skipped_bytes_cap"); + verify(routingPolicyEngine, never()).evaluate(any(), any(), any()); + verify(reviewPlanLookupService, never()).findForDatasource(any()); + } + + @Test + void aMissingEstimateUnderRejectRejectsEvenWhenAiFailed() { + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.FAILED, null, -1, + List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REJECTED), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.BYTES_CAP_REJECTED); + assertThat(step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP).reasonKey()) + .isEqualTo("workflow.decision.bytes_cap.no_estimate_rejected"); + } + + @Test + void anEstimateWithinTheCapChangesNothing() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of(), cap(5L, BytesScannedCapOutcome.WITHIN), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.PLAN_APPROVED); + assertThat(decision.bytesCap()).isNotNull(); + assertThat(decision.bytesCapChangedOutcome()).isFalse(); + assertThat(step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP).outcome()) + .isEqualTo(StepOutcome.ALLOW); + } + + @Test + void aMissingEstimateUnderRequireReviewSuppressesRoutingAutoApprove() { + givenPlan(false, true); + givenPolicyMatch(RoutingAction.AUTO_APPROVE, null); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), + clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.ROUTING_AUTO_APPROVE_SUPPRESSED); + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.PENDING_REVIEW); + assertThat(decision.sqlReviewSuppression()).isNull(); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + var routing = step(decision.trace(), QueryDecisionStepKind.ROUTING_POLICIES); + assertThat(routing.reasonKey()).isEqualTo( + "workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap"); + assertThat(routing.details()).containsEntry("bytes_cap_suppressed", true) + .containsEntry("sql_review_suppressed", false); + assertThat(step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP).outcome()) + .isEqualTo(StepOutcome.MATCH); + } + + @Test + void aMissingEstimateUnderRequireReviewSuppressesTheGrantAndThePlan() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenActiveGrant(grant(true, false, false, List.of(), List.of())); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), clock); + + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.PENDING_REVIEW); + assertThat(decision.sqlReviewSuppression()).isNull(); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + assertThat(step(decision.trace(), QueryDecisionStepKind.GRANT_FAST_PATH).reasonKey()) + .isEqualTo("workflow.decision.grant.suppressed_bytes_cap"); + var plan = step(decision.trace(), QueryDecisionStepKind.REVIEW_PLAN); + assertThat(plan.reasonKey()).isEqualTo("workflow.decision.plan.suppressed_bytes_cap"); + assertThat(plan.details()).containsEntry("bytes_cap_suppressed", true); + } + + @Test + void aMissingEstimateOnARequestAlreadyHeadedToReviewChangesNothing() { + givenPlan(false, true); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.UPDATE), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), + clock); + + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.PENDING_REVIEW); + assertThat(decision.bytesCapChangedOutcome()).isFalse(); + } + + @Test + void aMissingEstimateNeverSoftensAnAutoReject() { + givenPlan(false, true); + givenPolicyMatch(RoutingAction.AUTO_REJECT, null); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), + clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.ROUTING_AUTO_REJECT); + assertThat(decision.bytesCapChangedOutcome()).isFalse(); + } + + @Test + void anUnevaluatedCapIsReportedWithoutDecidingAnything() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of(), cap(null, null), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.PLAN_APPROVED); + var capStep = step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP); + assertThat(capStep.outcome()).isEqualTo(StepOutcome.SKIP); + assertThat(capStep.reasonKey()).isEqualTo("workflow.decision.bytes_cap.unevaluated"); + } + + @Test + void theSqlReviewNamesTheSuppressionWhenBothGuardsApply() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of("select_star"), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), clock); + + assertThat(decision.sqlReviewSuppression()).isNotNull(); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + assertThat(step(decision.trace(), QueryDecisionStepKind.REVIEW_PLAN).reasonKey()) + .isEqualTo("workflow.decision.plan.suppressed_sql_review"); + } + // ── Fixtures ────────────────────────────────────────────────────────────── private static com.bablsoft.accessflow.core.api.DecisionTraceStep step( diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java index 869667c53..5afc2b887 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java @@ -72,6 +72,9 @@ class QueryReviewStateMachineTest { @Mock AuditLogService auditLogService; @Mock MessageSource messageSource; @Mock ApplicationEventPublisher eventPublisher; + @Mock com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; + @Mock com.bablsoft.accessflow.proxy.api.QueryCostEstimateService queryCostEstimateService; + @Mock org.springframework.transaction.PlatformTransactionManager transactionManager; // A real ConditionContextFactory over the same mocks, not a mock of it: the context builder is // what turns these signals into routing input, and mocking it away would stop testing that. @@ -97,7 +100,8 @@ void buildStateMachine() { sqlParserService, routingPolicyEngine, accessGrantLookupService); stateMachine = new QueryReviewStateMachine(queryRequestLookupService, evaluator, queryRequestStateService, routingDecisionService, sqlReviewFindingService, - auditLogService, messageSource, eventPublisher); + auditLogService, messageSource, eventPublisher, bytesScannedCapResolutionService, + queryCostEstimateService, queryEstimateLookupService, transactionManager); } @BeforeEach @@ -691,6 +695,176 @@ void anAuditWriteFailureDoesNotUndoTheTransition() { QueryStatus.PENDING_REVIEW); } + // ── Bytes-scanned cap (#941) ────────────────────────────────────────────── + + private void givenCap(long limit, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + when(bytesScannedCapResolutionService.resolve(datasourceId, submitterId)) + .thenReturn(Optional.of(new com.bablsoft.accessflow.core.api.AppliedBytesCap(limit, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, missing))); + } + + private void givenEstimate(Long bytes, boolean failed) { + when(queryEstimateLookupService.findByQueryRequestId(queryId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), + queryId, "bigquery", QueryType.SELECT, !failed, null, null, null, null, + bytes, null, null, null, failed, null, 5, Instant.now()))); + } + + @Test + void aSkippedAnalysisWaitsForTheEstimateBeforeRouting() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, true, RiskLevel.LOW); + + stateMachine.onAiSkipped(new AiAnalysisSkippedEvent(queryId, "ai_analysis_enabled=false")); + + verify(queryCostEstimateService).estimateSubmittedQuery(queryId); + verify(queryRequestStateService, never()).recordBytesScannedCap(any(), org.mockito.ArgumentMatchers.anyLong(), + any(), any()); + } + + @Test + void aLostEstimateRaceIsReadBackRatherThanFailingTheDecision() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + // The listener won the insert: our own write fails, its row is what the cap compares. + when(queryCostEstimateService.estimateSubmittedQuery(queryId)) + .thenThrow(new org.springframework.dao.DataIntegrityViolationException("duplicate")); + givenEstimate(5_000L, false); + when(messageSource.getMessage(eq("workflow.bytes_cap.rejected_exceeded"), any(), any())) + .thenReturn("over the cap"); + + stateMachine.onAiSkipped(new AiAnalysisSkippedEvent(queryId, "ai_analysis_enabled=false")); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + verify(transactionManager).rollback(any()); + } + + @Test + void theAiCompletedPathOnlyEstimatesWhenACapApplies() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, true, RiskLevel.LOW); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryCostEstimateService, never()).estimateSubmittedQuery(any()); + } + + @Test + void anEstimateFailureOnTheSkippedPathIsAMissingEstimateNotAnError() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, true, RiskLevel.LOW); + when(queryCostEstimateService.estimateSubmittedQuery(queryId)) + .thenThrow(new IllegalStateException("warehouse down")); + + stateMachine.onAiSkipped(new AiAnalysisSkippedEvent(queryId, "ai_analysis_enabled=false")); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.PENDING_REVIEW); + } + + @Test + void anEstimateOverTheCapIsRejectedStampedAndAudited() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenEstimate(5_000L, false); + when(messageSource.getMessage(eq("workflow.bytes_cap.rejected_exceeded"), any(), any())) + .thenReturn("over the cap"); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + verify(eventPublisher).publishEvent(new QueryAutoRejectedEvent(queryId, null, "over the cap")); + var audit = org.mockito.ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(audit.capture()); + assertThat(audit.getValue().action()).isEqualTo(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED); + assertThat(audit.getValue().metadata()) + .containsEntry("stage", "decision") + .containsEntry("limit", 1_000L) + .containsEntry("estimated_bytes", 5_000L) + .containsEntry("outcome", "EXCEEDED") + .containsEntry("source", "DATASOURCE"); + verify(routingPolicyEngine, never()).evaluate(any(), any(), any()); + } + + @Test + void aFailedEstimateUnderRejectIsRejectedWithTheNoEstimateReason() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + givenEstimate(null, true); + when(messageSource.getMessage(eq("workflow.bytes_cap.rejected_no_estimate"), any(), any())) + .thenReturn("no estimate"); + + stateMachine.onAiFailed(new AiAnalysisFailedEvent(queryId, "provider error")); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + verify(eventPublisher).publishEvent(new QueryAutoRejectedEvent(queryId, null, "no estimate")); + var audit = org.mockito.ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(audit.capture()); + assertThat(audit.getValue().metadata()).doesNotContainKey("estimated_bytes") + .containsEntry("outcome", "NO_ESTIMATE_REJECTED"); + } + + @Test + void aMissingEstimateUnderRequireReviewHoldsAnAutoApprovalAndAuditsIt() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, false, RiskLevel.LOW); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.NO_ESTIMATE_REVIEW); + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.PENDING_REVIEW); + var audit = org.mockito.ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(audit.capture()); + assertThat(audit.getValue().action()).isEqualTo(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED); + } + + @Test + void anEstimateWithinTheCapIsStampedButNotAudited() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, false, RiskLevel.LOW); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + givenEstimate(10L, false); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.WITHIN); + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.APPROVED); + verify(auditLogService, never()).record(any()); + } + + @Test + void aFailingCapAuditNeverUndoesTheRejection() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenEstimate(5_000L, false); + org.mockito.Mockito.doThrow(new IllegalStateException("audit down")) + .when(auditLogService).record(any()); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + } + private void givenActiveGrant(AccessGrantView grant) { when(accessGrantLookupService.findActivePreApprovedGrants(organizationId, submitterId, datasourceId)).thenReturn(List.of(grant)); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java index b9fc2cfe8..a4f61cb8b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java @@ -69,7 +69,7 @@ private com.bablsoft.accessflow.core.api.QueryEstimateSnapshot estimate( Long estimatedRows, Long affectedRowCount, String scanType, boolean failed) { return new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), queryId, "postgresql", QueryType.SELECT, true, estimatedRows, affectedRowCount, - scanType, null, null, null, null, failed, null, 5, SUBMITTED_AT); + scanType, null, null, null, null, null, failed, null, 5, SUBMITTED_AT); } private QueryCorpusRow row(RiskLevel level, Integer score) { @@ -146,6 +146,23 @@ void historicalRowReplaysThePersistedCostEstimate() { assertThat(context.scanType()).isEqualTo("Seq Scan"); } + @Test + void theBytesEstimateIsReplayedAndAbsentOnAFailedEstimate() { + var withBytes = new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot( + UUID.randomUUID(), queryId, "bigquery", QueryType.SELECT, true, 10L, null, null, + null, 7_000_000L, null, null, null, false, null, 5, SUBMITTED_AT); + when(queryEstimateLookupService.findByQueryRequestId(queryId)) + .thenReturn(Optional.of(withBytes)); + + assertThat(factory.forHistoricalRow(row(RiskLevel.LOW, 10), ZoneId.of("UTC")) + .estimatedBytesScanned()).isEqualTo(7_000_000L); + + when(queryEstimateLookupService.findByQueryRequestId(queryId)) + .thenReturn(Optional.of(estimate(5_000L, null, "Seq Scan", true))); + assertThat(factory.forHistoricalRow(row(RiskLevel.LOW, 10), ZoneId.of("UTC")) + .estimatedBytesScanned()).isNull(); + } + @Test void historicalRowPrefersTheExactAffectedRowCountOverThePlanEstimate() { when(queryEstimateLookupService.findByQueryRequestId(queryId)) diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java index 12515eafa..6ffdd7627 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java @@ -52,6 +52,7 @@ void roundTripsEveryLeafAndCombinator() { new ConditionNode.TimeSinceLastApproval(ComparisonOperator.GT, 1440), new ConditionNode.CiCdOrigin(true), new ConditionNode.EstimatedRows(ComparisonOperator.GT, 100_000L), + new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, 1_000_000_000L), new ConditionNode.ScanTypeMatches(List.of("Seq*", "COLLSCAN")))); var json = codec.encode(tree); @@ -70,6 +71,17 @@ void estimatedRowsAndScanTypeUseSnakeCaseDiscriminators() { .contains("\"type\":\"scan_type\""); } + @Test + void estimatedBytesScannedUsesItsSnakeCaseDiscriminator() { + var json = codec.encode( + new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GTE, 1_000L)); + + assertThat(json).contains("\"type\":\"estimated_bytes_scanned\""); + assertThat(codec.decode( + "{\"type\":\"estimated_bytes_scanned\",\"operator\":\"GT\",\"value\":5}")) + .isEqualTo(new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, 5L)); + } + @Test void queryShapeUsesItsDiscriminatorAndUpperCaseShapeNames() { var json = codec.encode(new ConditionNode.QueryShapeIn(Set.of(QueryShape.GROUP_BY))); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java index 8abd7b199..709a845cc 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java @@ -343,6 +343,28 @@ void estimatedRowsFailsClosedWhenNoEstimate() { new ConditionNode.EstimatedRows(ComparisonOperator.GTE, 0), ctx)).isFalse(); } + @Test + void estimatedBytesScanned() { + var ctx = bytesContext(2_000_000_000_000L); + assertThat(evaluator.matches(new ConditionNode.EstimatedBytesScanned( + ComparisonOperator.GT, 1_000_000_000_000L), ctx)).isTrue(); + assertThat(evaluator.matches(new ConditionNode.EstimatedBytesScanned( + ComparisonOperator.LTE, 1_000_000_000_000L), ctx)).isFalse(); + } + + @Test + void estimatedBytesScannedFailsClosedWhenNoEstimate() { + assertThat(evaluator.matches(new ConditionNode.EstimatedBytesScanned( + ComparisonOperator.GTE, 0), bytesContext(null))).isFalse(); + } + + private ConditionContext bytesContext(Long bytes) { + return new ConditionContext(QueryType.SELECT, Set.of("ds.events"), RiskLevel.LOW, 10, + "ANALYST", Set.of(groupId), LocalDateTime.of(2026, 6, 3, 14, 30), + false, false, false, null, null, false, null, false, 10L, null, Set.of(), false, + bytes); + } + @Test void scanType() { var ctx = estimateContext(500L, "Seq Scan"); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java index 17188ff1f..f7e7d56be 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java @@ -119,7 +119,7 @@ void anAdminGetsTheFullOrderedTrace() { .exchange(); assertThat(result).hasStatus(200); - assertThat(result).bodyJson().extractingPath("$.steps.length()").asNumber().isEqualTo(12); + assertThat(result).bodyJson().extractingPath("$.steps.length()").asNumber().isEqualTo(13); assertThat(result).bodyJson().extractingPath("$.steps[0].step").asString() .isEqualTo("DATASOURCE_GATES"); // #864: the SQL review verdict sits between the permission gate and routing. @@ -127,7 +127,10 @@ void anAdminGetsTheFullOrderedTrace() { .isEqualTo("SQL_REVIEW"); assertThat(result).bodyJson().extractingPath("$.steps[4].outcome").asString() .isEqualTo("NO_MATCH"); - assertThat(result).bodyJson().extractingPath("$.steps[11].step").asString() + // #941: the bytes-scanned cap follows it; no cap configured here. + assertThat(result).bodyJson().extractingPath("$.steps[5].step").asString() + .isEqualTo("BYTES_SCANNED_CAP"); + assertThat(result).bodyJson().extractingPath("$.steps[12].step").asString() .isEqualTo("BREAK_GLASS"); assertThat(result).bodyJson().extractingPath("$.resulting_status").asString() .isEqualTo("PENDING_REVIEW"); @@ -292,11 +295,11 @@ void aDetailWithNoValueIsOmittedRatherThanSentAsNull() { .content(body(analyst.getId(), datasource.getId(), "SELECT id FROM orders")) .exchange(); - assertThat(result).bodyJson().extractingPath("$.steps[7].step").asString() + assertThat(result).bodyJson().extractingPath("$.steps[8].step").asString() .isEqualTo("REVIEW_PLAN"); - assertThat(result).bodyJson().doesNotHavePath("$.steps[7].details.review_plan_id"); - assertThat(result).bodyJson().doesNotHavePath("$.steps[7].details.min_approvals_required"); - assertThat(result).bodyJson().extractingPath("$.steps[7].details.requires_human_approval") + assertThat(result).bodyJson().doesNotHavePath("$.steps[8].details.review_plan_id"); + assertThat(result).bodyJson().doesNotHavePath("$.steps[8].details.min_approvals_required"); + assertThat(result).bodyJson().extractingPath("$.steps[8].details.requires_human_approval") .asBoolean().isFalse(); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java index 9c4d40922..518664c0b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java @@ -459,4 +459,34 @@ private QueryDetailView viewWithPrediction( null, Instant.now(), Instant.now()); } + + @Test + void fromCarriesTheBytesScannedCapAndTheBytesEstimate() { + var estimate = new QueryDetailView.CostEstimateDetail(UUID.randomUUID(), "bigquery", + QueryType.SELECT, true, null, null, null, null, null, null, null, false, null, 4, + 7_000L); + var base = new QueryDetailView(UUID.randomUUID(), UUID.randomUUID(), "Wh", DbType.BIGQUERY, + UUID.randomUUID(), UUID.randomUUID(), "a@x.io", "A", "SELECT 1", QueryType.SELECT, + QueryStatus.REJECTED, null, null, estimate, null, null, null, null, null, null, + null, null, null, null, List.of(), null, null, null, null, null, null, + Instant.now(), Instant.now(), null, null, null, null, + new QueryDetailView.BytesScannedCapDetail(1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED)); + + var response = QueryDetailResponse.from(base); + + assertThat(response.costEstimate().estimatedBytesScanned()).isEqualTo(7_000L); + assertThat(response.bytesScannedCap()).isEqualTo(new QueryDetailResponse.BytesScannedCapDetail( + 1_000L, com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED)); + } + + @Test + void theCompatibleCostEstimateConstructorLeavesTheBytesEstimateAbsent() { + var estimate = new QueryDetailView.CostEstimateDetail(UUID.randomUUID(), "postgresql", + QueryType.SELECT, true, 1L, null, null, null, null, null, null, false, null, 4); + + assertThat(estimate.estimatedBytesScanned()).isNull(); + } } diff --git a/docs/03-data-model.md b/docs/03-data-model.md index 48bf1fbe6..bb6d52c54 100644 --- a/docs/03-data-model.md +++ b/docs/03-data-model.md @@ -234,6 +234,8 @@ A customer database that AccessFlow proxies. Credentials are stored encrypted. | `api_key_encrypted` | TEXT NULL (AF-420, migration `V80`) — AES-256-GCM-encrypted API key for the search engines (`ELASTICSEARCH` / `OPENSEARCH`), sent as `Authorization: ApiKey`. `@JsonIgnore`, never returned in an API response. NULL for basic-auth search datasources and every other dialect; the service layer requires **either** `username`+`password` **or** `api_key` for a search datasource. May also hold an external secret reference (AF-448), same semantics as `password_encrypted`. | | `private_key_passphrase_encrypted` | TEXT NULL (#632, migration `V163`) — AES-256-GCM-encrypted passphrase for a passphrase-protected (encrypted) PKCS#8 private key held in `password_encrypted`, used by `SNOWFLAKE` key-pair auth. `@JsonIgnore`, never returned in an API response. NULL for password credentials, unencrypted PEMs, and every other dialect; the service layer rejects it for any `db_type` other than `SNOWFLAKE`. May also hold an external secret reference (AF-448), same semantics as `password_encrypted`. | | `environment` | ENUM `datasource_environment` NULL (#861, migration `V170`): `DEVELOPMENT` \| `TEST` \| `STAGING` \| `PRODUCTION`. Optional — an unset environment is a legitimate state. It keys per-environment SQL review severity: a datasource resolves to the [`sql_review_rulesets`](#sql-review-sqlreview-861--epic-860) row bound to its environment, else the organization-wide default (`environment IS NULL`), else no rules. Existing rows stay NULL. Set on create, and on update via `environment` (null = unchanged) / `clear_environment` (unset). | +| `max_bytes_scanned_per_query` | BIGINT nullable, `CHECK > 0` (#941, V192) — the **bytes-scanned cap**: the largest pre-flight scan estimate a query on this datasource may have, in raw bytes. NULL = no cap. Configurable only on the engines whose dry-run reports a bytes estimate — BigQuery, Snowflake, Databricks (`core.api.BytesScannedCapSupport`); any other engine is refused with 422 `BYTES_SCANNED_CAP_NOT_SUPPORTED`. Set on create; on update `max_bytes_scanned_per_query` (null = unchanged) / `clear_max_bytes_scanned_per_query` (remove). Enforced when the query leaves `PENDING_AI` and again just before execution — see [05-backend.md → Bytes-scanned cost caps](05-backend.md#bytes-scanned-cost-caps-941) | +| `bytes_cap_missing_estimate` | ENUM `bytes_cap_missing_estimate_action` NOT NULL DEFAULT `REQUIRE_REVIEW` (#941, V192) — what the cap does when a query has **no** bytes estimate (the estimate failed, DDL, a shape the engine cannot plan): `REQUIRE_REVIEW` suppresses every automatic approval so a person decides; `REJECT` refuses the query. Inert while no cap applies | | `is_active` | BOOLEAN DEFAULT true | | `created_at` | TIMESTAMPTZ | @@ -307,6 +309,7 @@ Grants a specific user access to a specific datasource with granular controls. | `can_ddl` | BOOLEAN DEFAULT false — CREATE/ALTER/DROP | | `can_break_glass` | BOOLEAN NOT NULL DEFAULT false (AF-385, Flyway V94) — grants the emergency break-glass submission mode on this datasource. Required for everyone (including admins); time-boxed via `expires_at`. | | `row_limit_override` | INTEGER nullable (API-validated ≥ 1) — lowers the datasource's `max_rows_per_query` for this grantee, never raises it; the global `ACCESSFLOW_PROXY_EXECUTION_MAX_ROWS` ceiling still applies. Across a user's direct and group grants the **smallest** non-null value wins (#933). Enforced on direct, scheduled, recurring, break-glass and grouped executions and on the table-preview endpoint | +| `bytes_scanned_limit_override` | BIGINT nullable, `CHECK > 0` (#941, V192) — a bytes-scanned cap for this grantee. Combined with the datasource's `max_bytes_scanned_per_query`, the **smallest** applies; across a user's direct and group grants the smallest non-null value wins, like `row_limit_override`. A JIT approval that replaces a time-boxed row carries the replaced row's value over, so an approval never widens it. Refused with 422 `BYTES_SCANNED_CAP_NOT_SUPPORTED` on an engine without a bytes estimate | | `allowed_schemas` | TEXT[] — null means all schemas permitted | | `allowed_tables` | TEXT[] — null means all tables permitted | | `restricted_columns` | TEXT[] nullable — fully-qualified `schema.table.column` entries whose values are masked in SELECT results before persistence and surfaced to the AI analyzer; null/empty means no column restrictions. A column listed here with no matching `masking_policy` row uses the static `FULL` mask (`***`); a `masking_policy` for the same column overrides it with the configured strategy. | @@ -330,8 +333,8 @@ constraint and restriction columns clean), keyed on `group_id` instead of `user_ grant they belong to — resolved in `DefaultDatasourceUserPermissionLookupService` (flags OR-ed; allow-lists unioned; `restricted_columns` intersected so a column is masked only when every contributing grant masks it; each grant's `expires_at` honoured independently). Two deliberate inversions: -`row_limit_override` merges to the **smallest** non-null value so a wide group grant can never -raise a tight per-user cap (#933), and the deny-lists — `denied_schemas` / `denied_tables` (#939), +`row_limit_override` and `bytes_scanned_limit_override` (#941) merge to the **smallest** non-null +value so a wide group grant can never raise a tight per-user cap (#933), and the deny-lists — `denied_schemas` / `denied_tables` (#939), `denied_columns` (#1099) and `denied_shapes` (#940) — merge to their **union**, so a permissive grant can never lift another grant's denial and a group grant's denial binds every member. A denial lives on its row, so revoking or expiring that row (including an attestation revoke) drops the denial @@ -346,6 +349,7 @@ masking-reveal and row-security. | `group_id` | FK → `user_groups` ON DELETE CASCADE | | `can_read` / `can_write` / `can_ddl` / `can_break_glass` | BOOLEAN NOT NULL DEFAULT false — same semantics as the per-user table | | `row_limit_override` | INTEGER nullable — same semantics as the per-user table; merged most-restrictive (smallest non-null wins) | +| `bytes_scanned_limit_override` | BIGINT nullable (#941, V192) — same semantics as the per-user table; merged most-restrictive | | `allowed_schemas` / `allowed_tables` / `restricted_columns` / `denied_columns` / `denied_schemas` / `denied_tables` / `denied_shapes` | TEXT[] nullable — same semantics as the per-user table (`denied_columns` added by V186, #935; `denied_schemas` / `denied_tables` by V190, #939; `denied_shapes` by V191, #940 — deny-lists merge as a union across a user's grants) | | `expires_at` | TIMESTAMPTZ nullable — honoured per grant (an expired grant contributes nothing) | | `created_by` | FK → `users` | @@ -828,6 +832,7 @@ The condition is a polymorphic, `"type"`-discriminated tree (snake_case, no exte | `cicd_origin` (AF-446) | `expected: bool` | whether the request came from a CI/CD pipeline (submitted via an API key or with the `X-AccessFlow-CI` header) equals `expected`. Deterministic — the flag defaults to `false` | | `anomaly_detected` (AF-383) | `expected: bool` | whether the submitter currently has an `OPEN` `behavior_anomaly` on the target datasource equals `expected`. The UBA detector is a periodic batch over **past** data, so this signal escalates the flagged user's **next** query — pair it with `ESCALATE`. Deterministic — false when the user has no open anomaly there | | `estimated_rows` (AF-624) | `operator` (`LT`/`LTE`/`GT`/`GTE`/`EQ`), `value` | the query's pre-flight estimated row impact (the exact affected-row count for UPDATE/DELETE when available, else the EXPLAIN estimate) satisfies the comparison. Read live from `query_estimates` at routing time. **Fails closed**: false when no estimate signal exists (not yet computed, unsupported engine, or failed) | +| `estimated_bytes_scanned` (#941) | `operator` (`LT`/`LTE`/`GT`/`GTE`/`EQ`), `value` (raw bytes) | the warehouse's pre-flight bytes-scanned estimate (`query_estimates.estimated_bytes_scanned`) satisfies the comparison. An advisory escalation trigger — the hard cap is `datasources.max_bytes_scanned_per_query`. **Fails closed**: false when no bytes estimate exists (every engine other than BigQuery / Snowflake / Databricks, and a failed or unsupported estimate) | | `scan_type` (AF-624) | `patterns: [string]` | the pre-flight plan's root operation (e.g. `Seq Scan`, `COLLSCAN`) matches any glob (`*` wildcard, case-insensitive). **Fails closed**: false when no plan was captured | On the AI-skipped path (`datasource.ai_analysis_enabled=false`) the risk-based operands (`risk_level`, `risk_score`) evaluate to **false** — there is no AI signal. Routing is **not** run on the AI-failure path. @@ -1056,6 +1061,9 @@ The central entity. Represents a single SQL submission through the platform. | `cicd_origin` | BOOLEAN NOT NULL DEFAULT FALSE (AF-446, `V88`) — true when the query was submitted via an API key or with the `X-AccessFlow-CI` header. Read by the `cicd_origin` routing condition. | | `application_name` | VARCHAR(100) nullable (#938, V189) — the calling application, resolved at submission by `security.api.RequestApplicationService`: the presenting API key's `application_name` when it has one, else the sanitised `X-AccessFlow-Application` header (trimmed; dropped when blank or containing control characters; truncated to 100 chars), else NULL. Copied onto recurring occurrences. Partial index `idx_query_requests_application_name ON query_requests(application_name) WHERE application_name IS NOT NULL` backs the `application_name` list filter. Identification and audit only — **not** a routing operand (a future operand must fail closed on a `HEADER` source, like `cicd_origin`'s header path) | | `application_name_source` | ENUM `application_name_source`: `API_KEY` \| `HEADER`, nullable (#938, V189) — where `application_name` came from. `API_KEY` is trustworthy; `HEADER` is entirely client-controlled and the UI labels it *Untrusted*. NULL exactly when `application_name` is NULL | +| `bytes_scanned_cap` | BIGINT nullable (#941, V192) — the bytes-scanned cap that bound the submitter when the query left `PENDING_AI`: the smaller of the datasource cap and the merged grant override. Stamped by `QueryReviewStateMachine` before the decision is applied, whatever the outcome; NULL when no cap applied. Surfaced as `bytes_scanned_cap` on `GET /queries/{id}` | +| `bytes_scanned_cap_source` | ENUM `bytes_scanned_cap_source`: `DATASOURCE` \| `GRANT`, nullable (#941) — which setting supplied the cap; on a tie the datasource. NULL exactly when `bytes_scanned_cap` is NULL | +| `bytes_scanned_cap_outcome` | ENUM `bytes_scanned_cap_outcome`: `WITHIN` \| `EXCEEDED` \| `NO_ESTIMATE_REVIEW` \| `NO_ESTIMATE_REJECTED`, nullable (#941) — how the persisted estimate compared. `EXCEEDED` and `NO_ESTIMATE_REJECTED` accompany a `PENDING_AI → REJECTED` transition. The re-check just before execution does not update it: a refusal there is recorded as the `FAILED` row's `error_message` | | `approved_by_grant_id` | UUID nullable (#582, `V112`) — id of the `access_grant_request` whose pre-approval fast-path auto-approved this query. Bare UUID (no FK, mirroring `granted_permission_id`): the grant's lifecycle (expiry, revocation) is independent of the query's audit trail. Stamped atomically with the `PENDING_AI → APPROVED` transition by `QueryRequestStateService.approveByAccessGrant`; surfaced as `approved_by_grant` on `GET /queries/{id}`. | | `created_at` | TIMESTAMPTZ | | `updated_at` | TIMESTAMPTZ | @@ -1069,6 +1077,10 @@ PENDING_AI → PENDING_REVIEW → APPROVED → EXECUTED ↘ PENDING_REVIEW (if no AI) ↘ APPROVED (routing policy AUTO_APPROVE — see routing_policy) ↘ REJECTED (routing policy AUTO_REJECT — see routing_policy) + ↘ REJECTED (bytes-scanned cap, #941 — estimate over the cap, or none under + bytes_cap_missing_estimate=REJECT; decided before routing) + ↘ PENDING_REVIEW (bytes-scanned cap with no estimate under REQUIRE_REVIEW — every + auto-approve path is held for a person, like a SQL review BLOCK) PENDING_REVIEW → CANCELLED (by submitter) APPROVED → CANCELLED (submitter, when scheduled_for is set and run hasn't fired yet; for a recurring series — recurrence_rule set, #627 — also any @@ -1156,7 +1168,7 @@ Stores the result of an AI analysis run for a query request. ## query_estimates -Persisted pre-flight cost / blast-radius estimate for a submitted query (AF-624, Flyway `V127`/`V128`). Computed asynchronously right after submission by the proxy's `QueryCostEstimateService` — the AF-445 dry-run machinery (relational dialect `EXPLAIN`, engine `dryRun` overrides) plus, for UPDATE/DELETE, a governed non-mutating affected-row count (relational `SELECT COUNT(*)` rewrite; MongoDB `countDocuments`, SQL++ `SELECT COUNT(*)`, Cypher `MATCH … RETURN count(*)`, Elasticsearch `_count` via the engines' `countAffectedRows` SPI overrides). One row per query request; every computation path persists a row (success, unsupported-engine degrade, or `failed=true` sentinel), mirroring `ai_analyses`. Read by reviewers (`GET /queries/{id}` → `cost_estimate`), by the routing-policy engine (`estimated_rows` / `scan_type` conditions), and folded into the AI analyzer's prompt (`{{cost_estimate}}`). +Persisted pre-flight cost / blast-radius estimate for a submitted query (AF-624, Flyway `V127`/`V128`). Computed asynchronously right after submission by the proxy's `QueryCostEstimateService` — the AF-445 dry-run machinery (relational dialect `EXPLAIN`, engine `dryRun` overrides) plus, for UPDATE/DELETE, a governed non-mutating affected-row count (relational `SELECT COUNT(*)` rewrite; MongoDB `countDocuments`, SQL++ `SELECT COUNT(*)`, Cypher `MATCH … RETURN count(*)`, Elasticsearch `_count` via the engines' `countAffectedRows` SPI overrides). One row per query request; every computation path persists a row (success, unsupported-engine degrade, or `failed=true` sentinel), mirroring `ai_analyses`. Read by reviewers (`GET /queries/{id}` → `cost_estimate`), by the routing-policy engine (`estimated_rows` / `estimated_bytes_scanned` / `scan_type` conditions), by the bytes-scanned cap (#941), and folded into the AI analyzer's prompt (`{{cost_estimate}}`). When AI analysis is skipped, the state machine computes it (insert-once) before routing, so a condition never races the asynchronous listener. | Column | Type / Notes | |--------|-------------| @@ -1169,6 +1181,7 @@ Persisted pre-flight cost / blast-radius estimate for a submitted query (AF-624, | `affected_row_count` | BIGINT nullable — exact governed count for UPDATE/DELETE; null when the shape can't be provably counted (joins, `USING`, MERGE, …) or the engine doesn't support counting | | `scan_type` | VARCHAR(128) nullable — the plan's root operation (e.g. `Seq Scan`, `COLLSCAN`) | | `estimated_cost` | DOUBLE PRECISION nullable — the plan's root cost when the engine exposes one | +| `estimated_bytes_scanned` | BIGINT nullable (#941, V192) — the warehouse engine's native pre-flight scan estimate in raw bytes (AF-634: BigQuery dry-run `totalBytesProcessed`, Snowflake `bytesAssigned`, Databricks `sizeInBytes`); NULL for every other engine. Read by the `estimated_bytes_scanned` routing condition and the bytes-scanned cap | | `plan` | JSONB nullable — the snake_case plan-node tree (same shape as the dry-run endpoint's `plan`). Every node's `detail` is null when row security applied to the submitter, because engines inline the bound values into predicate text (#1092; V188 stripped all rows stored earlier) | | `raw_plan` | TEXT nullable — the engine's raw plan output; null when row security applied (#1092) | | `unsupported_reason` | VARCHAR(500) nullable — localized reason when `supported=false` | @@ -1763,6 +1776,7 @@ The hash chain (added in V26) is per organization. Inserts are serialized by a P | `SERVICE_ACCOUNT_DELEGATION_GRANTED` / `SERVICE_ACCOUNT_DELEGATION_REVOKED` | A human (`channel=self_service`, via `/me/service-account-delegations`) or an admin (`channel=admin`) let a service account act on a human's behalf, or revoked that grant (#874). Resource: `service_account`. Metadata: `delegation_id`, `principal_user_id`, `expires_at` (when set), `channel`. | | `SERVICE_ACCOUNT_KEY_ISSUED` / `SERVICE_ACCOUNT_KEY_ROTATED` / `SERVICE_ACCOUNT_KEY_REVOKED` | Admin issues / rotates / revokes an API key on behalf of a service account (#871). Resource: `service_account`. Metadata: `api_key_id`, `name` (issue); `api_key_id`, `superseded_key_id`, `name`, `superseded_expires_at` (rotate — the old key's new expiry, i.e. the end of the grace window); `api_key_id` (revoke). The raw key is never logged. | | `SQL_REVIEW_BLOCKED` | A `BLOCK` SQL review finding suppressed an auto-approve path and forced the request to human review (#864). System-attributed: `actor_id` is NULL. Resource: `query_request` (written by `QueryReviewStateMachine` after the transition) or `request_group` (written by `GroupAiAnalysisListener`). Metadata: `trigger: "sql_review"`, `blocking_rule_ids` (distinct, sorted), `suppressed_paths` — one or more of `ROUTING_AUTO_APPROVE`, `GRANT_FAST_PATH`, `REVIEW_PLAN` for a query, `GROUP_REVIEW_PLAN` for a group — plus `matched_policy_id` when routing was the suppressed path and `blocking_item_ids` for a group. Written **only when the guard changed the outcome**: never for `WARN`, never on a routing `AUTO_REJECT` (the rejection stands), never on the AI-failed path or a plan that already required review (the findings are still on the detail), and never for break-glass, which records findings but is not gated. | +| `QUERY_BYTES_SCANNED_CAP_ENFORCED` | The bytes-scanned cap (#941) changed a query's outcome. System-attributed: `actor_id` is NULL. Resource: `query_request` — or `request_group` when a group member was refused at execution (metadata then carries `item_id`). Metadata: `trigger: "bytes_scanned_cap"`, `stage` (`decision` — the query left `PENDING_AI` rejected, or an automatic approval was held for review; `execution` — refused just before running, the query is then `FAILED`), `limit`, `source` (`DATASOURCE` \| `GRANT`), `outcome`, and `estimated_bytes` when an estimate existed. Never written when the estimate was within the cap; a successful `QUERY_EXECUTED` row under a cap instead carries `bytes_scanned_cap`, `bytes_scanned_cap_source` and `bytes_scanned_estimate` | | `MASKING_POLICY_CREATED` / `MASKING_POLICY_UPDATED` / `MASKING_POLICY_DELETED` | Admin creates / updates / deletes a masking policy via the `/datasources/{id}/masking-policies` CRUD endpoints. Resource: `masking_policy`. | | `ROW_SECURITY_POLICY_CREATED` / `ROW_SECURITY_POLICY_UPDATED` / `ROW_SECURITY_POLICY_DELETED` | Admin creates / updates / deletes a row-security policy via the `/datasources/{id}/row-security-policies` CRUD endpoints (AF-380). Resource: `row_security_policy`. Applied row-security policy ids at execute time ride on `QUERY_EXECUTED` metadata (`applied_row_security_policy_ids`), not a separate action. | | `ROW_LIMIT_POLICY_CREATED` / `ROW_LIMIT_POLICY_UPDATED` / `ROW_LIMIT_POLICY_DELETED` | Admin creates / updates / deletes a per-table row-limit policy via the `/datasources/{id}/row-limit-policies` CRUD endpoints (#934). Resource: `row_limit_policy`. Metadata carries `datasource_id`, `schema_name`, `table_name`, `max_rows`, `enabled`. The lowest-cap matching policies of a SELECT ride on `QUERY_EXECUTED` metadata (`applied_row_limit_policy_ids`) when their cap was the binding one. | diff --git a/docs/04-api-spec.md b/docs/04-api-spec.md index 43e8309de..cb4aad957 100644 --- a/docs/04-api-spec.md +++ b/docs/04-api-spec.md @@ -397,10 +397,22 @@ Results are scoped to the caller's organization. ADMINs see all datasources in t "read_replicas": [], "result_cache_enabled": false, "result_cache_ttl_seconds": null, - "environment": "PRODUCTION" + "environment": "PRODUCTION", + "max_bytes_scanned_per_query": null, + "bytes_cap_missing_estimate": "REQUIRE_REVIEW" } ``` +`max_bytes_scanned_per_query` (#941) is the optional **bytes-scanned cap** in raw bytes (`@Min(1)`, +`validation.bytes_cap.min`): a query whose pre-flight scan estimate is larger is refused. It is +accepted only for `BIGQUERY`, `SNOWFLAKE` and `DATABRICKS` — the engines whose dry-run reports a +bytes estimate — and any other `db_type` answers **422** `BYTES_SCANNED_CAP_NOT_SUPPORTED` (with +`dbType`). `bytes_cap_missing_estimate` (`REQUIRE_REVIEW` default, or `REJECT`) decides what the +cap does with a query that has no bytes estimate: hold every automatic approval for a person, or +refuse the query. It is stored on every datasource but only matters while a cap applies — see +[05-backend.md → Bytes-scanned cost caps](05-backend.md#bytes-scanned-cost-caps-941). Both are +returned on the datasource object; an unset cap is omitted. + `environment` (#861, epic #860) is optional: one of `DEVELOPMENT` / `TEST` / `STAGING` / `PRODUCTION`, or omitted. It selects which SQL review ruleset the datasource's queries are evaluated against — the ruleset bound to that environment, else the organization-wide default. An @@ -473,10 +485,17 @@ All fields optional. Omitted fields are left unchanged. Providing `password` tri "result_cache_ttl_seconds": 120, "environment": "STAGING", "clear_environment": false, + "max_bytes_scanned_per_query": 1000000000000, + "clear_max_bytes_scanned_per_query": false, + "bytes_cap_missing_estimate": "REJECT", "active": true } ``` +`max_bytes_scanned_per_query` (#941) follows the same shape: omitted or `null` leaves the cap +unchanged, `"clear_max_bytes_scanned_per_query": true` removes it, and a value wins when both are +sent. Setting a cap on an engine without a bytes estimate is **422** `BYTES_SCANNED_CAP_NOT_SUPPORTED`. + `environment` follows the `clear_ai_config` shape (#861): omitted or `null` leaves it unchanged, `"clear_environment": true` unsets it, and a non-null `environment` wins when both are sent. @@ -485,6 +504,7 @@ All fields optional. Omitted fields are left unchanged. Providing `password` tri **Response 200:** Updated datasource object. **Response 404:** Datasource does not exist in the caller's organization. `error: DATASOURCE_NOT_FOUND`. **Response 409:** Renaming would conflict with another datasource in the same organization. `error: DATASOURCE_NAME_ALREADY_EXISTS`. +**Response 422:** A bytes-scanned cap on an engine without a bytes estimate. `error: BYTES_SCANNED_CAP_NOT_SUPPORTED`, with `dbType`. ### DELETE /datasources/{id} @@ -628,6 +648,7 @@ ADMINs may sample any datasource in their organization; non-ADMINs need a permis "can_write": false, "can_ddl": false, "row_limit_override": null, + "bytes_scanned_limit_override": null, "allowed_schemas": ["public"], "allowed_tables": null, "restricted_columns": ["public.users.ssn"], @@ -661,6 +682,7 @@ ADMINs may sample any datasource in their organization; non-ADMINs need a permis "can_ddl": false, "can_break_glass": false, "row_limit_override": 1000, + "bytes_scanned_limit_override": null, "allowed_schemas": ["public"], "allowed_tables": ["users", "orders"], "restricted_columns": ["public.users.ssn", "public.users.email"], @@ -691,6 +713,10 @@ It is stored in declaration order with duplicates dropped, recorded in the grant non-empty, and supported only on the in-process relational engines. `can_break_glass` (AF-385, optional, default `false`) grants the emergency break-glass submission mode on this datasource — time-boxed via `expires_at`. The flag is returned on the permission object alongside `can_read`/`can_write`/`can_ddl`. +`bytes_scanned_limit_override` (#941, optional, `@Min(1)`) is a bytes-scanned cap in raw bytes for +this grantee: the smaller of it and the datasource's `max_bytes_scanned_per_query` applies, and +across a user's direct and group grants the smallest wins. It is accepted only on BigQuery, +Snowflake and Databricks datasources (422 `BYTES_SCANNED_CAP_NOT_SUPPORTED` otherwise). **Response 201:** Permission object. `Location` header points to `/api/v1/datasources/{id}/permissions/{permId}`. **Response 404:** Datasource does not exist in the caller's organization. `error: DATASOURCE_NOT_FOUND`. @@ -698,6 +724,7 @@ default `false`) grants the emergency break-glass submission mode on this dataso **Response 422:** Target user does not exist or does not belong to the caller's organization. `error: ILLEGAL_DATASOURCE_PERMISSION`. **Response 422:** `denied_columns` is non-empty on an engine-managed datasource (every engine plugin, warehouses included). `error: DENIED_COLUMNS_NOT_SUPPORTED`, with `dbType`. **Response 422:** `denied_shapes` is non-empty on an engine-managed datasource. `error: DENIED_SHAPES_NOT_SUPPORTED`, with `dbType`. +**Response 422:** `bytes_scanned_limit_override` on an engine without a bytes estimate. `error: BYTES_SCANNED_CAP_NOT_SUPPORTED`, with `dbType`. ### DELETE /datasources/{id}/permissions/{permId} @@ -728,6 +755,7 @@ the group instead of a user: "can_ddl": false, "can_break_glass": false, "row_limit_override": null, + "bytes_scanned_limit_override": null, "allowed_schemas": ["public"], "allowed_tables": null, "restricted_columns": ["public.users.ssn"], @@ -755,6 +783,7 @@ Same body as the per-user grant with `group_id` in place of `user_id`: "can_ddl": false, "can_break_glass": false, "row_limit_override": 1000, + "bytes_scanned_limit_override": 500000000000, "allowed_schemas": ["public"], "allowed_tables": ["users", "orders"], "restricted_columns": ["public.users.ssn"], @@ -771,6 +800,7 @@ Same body as the per-user grant with `group_id` in place of `user_id`: **Response 409:** A permission row already exists for `(group_id, datasource_id)`. `error: DATASOURCE_GROUP_PERMISSION_ALREADY_EXISTS`. **Response 422:** `denied_columns` on an engine-managed datasource. `error: DENIED_COLUMNS_NOT_SUPPORTED`. **Response 422:** `denied_shapes` on an engine-managed datasource. `error: DENIED_SHAPES_NOT_SUPPORTED`. +**Response 422:** `bytes_scanned_limit_override` on an engine without a bytes estimate. `error: BYTES_SCANNED_CAP_NOT_SUPPORTED`. ### DELETE /datasources/{id}/permissions/groups/{permId} @@ -1815,8 +1845,10 @@ Each subsequent row contains the same fields as `QueryListItemView`. `ai_risk_le "unsupported_reason": null, "failed": false, "error_message": null, - "duration_ms": 12 + "duration_ms": 12, + "estimated_bytes_scanned": null }, + "bytes_scanned_cap": null, "approval_prediction": { "id": "uuid", "probability": 0.78, @@ -1890,7 +1922,9 @@ Each subsequent row contains the same fields as `QueryListItemView`. `ai_risk_le `matched_policy` is the routing policy that decided this query's routing (AF-379); `null` when no policy matched and the query fell through to the datasource's review plan. `policy_name` is `null` when the matched policy was later deleted. The frontend renders a "matched policy" alert on the detail page when this object is present. See [docs/05-backend.md → "Policy-as-code routing engine"](05-backend.md#policy-as-code-routing-engine-af-379). -`cost_estimate` is the query's persisted pre-flight cost / blast-radius estimate (AF-624), computed automatically and asynchronously right after submission — `null` while it is still being computed (typically only during `PENDING_AI`). `supported=false` with a localized `unsupported_reason` marks engines/statement shapes with no plan concept; `failed=true` with `error_message` marks a computation error. `affected_row_count` is the exact governed row count for UPDATE/DELETE (relational `SELECT COUNT(*)` rewrite, or the engine's native non-mutating count) and is `null` when the shape cannot be provably counted; `estimated_rows`/`scan_type`/`estimated_cost` come from the plan root, and `plan` reuses the dry-run endpoint's plan-node shape (when row security applied to the submitter, every node's `detail` and `raw_plan` are `null` so bound attribute values are never exposed — #1092) — see [POST /queries/dry-run](#post-queriesdry-run--response-200) and [docs/05-backend.md → "Automatic pre-flight cost estimate"](05-backend.md#automatic-pre-flight-cost-estimate-af-624). The `query.estimate_complete` WebSocket event signals completion. +`cost_estimate` is the query's persisted pre-flight cost / blast-radius estimate (AF-624), computed automatically and asynchronously right after submission — `null` while it is still being computed (typically only during `PENDING_AI`). `supported=false` with a localized `unsupported_reason` marks engines/statement shapes with no plan concept; `failed=true` with `error_message` marks a computation error. `affected_row_count` is the exact governed row count for UPDATE/DELETE (relational `SELECT COUNT(*)` rewrite, or the engine's native non-mutating count) and is `null` when the shape cannot be provably counted; `estimated_rows`/`scan_type`/`estimated_cost` come from the plan root, and `plan` reuses the dry-run endpoint's plan-node shape (when row security applied to the submitter, every node's `detail` and `raw_plan` are `null` so bound attribute values are never exposed — #1092) — see [POST /queries/dry-run](#post-queriesdry-run--response-200) and [docs/05-backend.md → "Automatic pre-flight cost estimate"](05-backend.md#automatic-pre-flight-cost-estimate-af-624). The `query.estimate_complete` WebSocket event signals completion. `estimated_bytes_scanned` (#941) is the warehouse engine's pre-flight scan estimate in raw bytes (BigQuery / Snowflake / Databricks) and is omitted for every other engine. + +`bytes_scanned_cap` (#941) is the bytes-scanned cap that bound the submitter when the query left `PENDING_AI` — `{ "limit": 1000000000000, "source": "DATASOURCE" | "GRANT", "outcome": "WITHIN" | "EXCEEDED" | "NO_ESTIMATE_REVIEW" | "NO_ESTIMATE_REJECTED" }`, omitted when no cap applied. `EXCEEDED` and `NO_ESTIMATE_REJECTED` accompany status `REJECTED`; `NO_ESTIMATE_REVIEW` means an automatic approval was held for a person. A refusal by the re-check just before execution does not change this object: the query is `FAILED` and `error_message` names the estimate and the cap. `approval_prediction` is the query's advisory approval-outcome prediction (AF-645) — the probability that a human reviewer approves it, computed from the organization's own decision history. It is a **triage signal only**: it never auto-approves, auto-rejects, or feeds the routing engine, grant coverage, or any other decision path, and clients must present it in neutral wording rather than as an instruction. The block is `null` until the asynchronous scoring pass persists a row (a query only ever gets one — auto-approved, auto-rejected, grant-covered and break-glass queries never reach review, so they never get scored at all). Once present, exactly one of three shapes applies: `probability` in `[0,1]` with `skipped=false, failed=false`; `skipped=true` with a `skipped_reason` machine token the client localizes — the closed set is `DISABLED` (the feature is switched off) and `MODEL_NOT_SERVING` (the org has too little history, the model failed its holdout quality gate, or it was trained against an older feature schema); or `failed=true`, the sentinel for an unexpected scoring error. `probability` is `null` on both sentinel shapes. The block is **omitted entirely** — not merely blanked — for callers who lack `QUERY_REVIEW` and for the query's own submitter, a reviewer reading their own request included: the prediction is a triage aid for whoever decides, and letting a submitter read the likely verdict on their own query would invite cancelling and resubmitting until it looks better. See [docs/05-backend.md → "Approval-outcome prediction"](05-backend.md#approval-outcome-prediction-af-645). The `query.prediction_complete` WebSocket event signals completion. @@ -4001,7 +4035,7 @@ All endpoints require `role=ADMIN` and operate within the caller's organization. } ``` -`name`, `condition`, and `action` are **required**. `datasource_id` is optional (null = org-wide). `priority` must be unique within the organization. `required_approvals` is required (and only meaningful) for `action: REQUIRE_APPROVALS` (absolute minimum approvers) and `action: ESCALATE` (delta added to the review-plan minimum, default 1); it must be null for `AUTO_APPROVE` / `AUTO_REJECT`. The `condition` is the typed `"type"`-discriminated tree documented in the data model — including the AF-446 client-context operands `source_ip` (CIDR allow-list; deny via `not`), `user_agent`, `time_since_last_approval`, and `cicd_origin`, which **fail closed** when their signal is absent. A malformed CIDR in a `source_ip` leaf is rejected with **422** `ROUTING_POLICY_INVALID`. The `query_shape` operand (#940) — `{"type": "query_shape", "any_of": ["JOIN", "SUBQUERY"]}` — matches a query that has any listed shape (`JOIN`, `UNION`, `SUBQUERY`, `CTE`, `GROUP_BY`, `HAVING`, `AGGREGATE`, `WINDOW_FUNCTION`) anywhere in the statement; an empty `any_of` is rejected with **422** `ROUTING_POLICY_INVALID`, and the leaf fails closed when the SQL cannot be parsed for its shape. Routing re-parses the stored SQL text with JSqlParser whatever the engine, so on a plugin datasource the leaf matches only when that text happens to be standard SQL JSqlParser can read (often true for a warehouse, never for a MongoDB or Redis command). Pair it with `ESCALATE` or `REQUIRE_APPROVALS` to send, say, every joined query to a second reviewer, or with `AUTO_REJECT` to refuse it outright; a grant's `denied_shapes` refuses at submission instead. +`name`, `condition`, and `action` are **required**. `datasource_id` is optional (null = org-wide). `priority` must be unique within the organization. `required_approvals` is required (and only meaningful) for `action: REQUIRE_APPROVALS` (absolute minimum approvers) and `action: ESCALATE` (delta added to the review-plan minimum, default 1); it must be null for `AUTO_APPROVE` / `AUTO_REJECT`. The `condition` is the typed `"type"`-discriminated tree documented in the data model — including the AF-446 client-context operands `source_ip` (CIDR allow-list; deny via `not`), `user_agent`, `time_since_last_approval`, and `cicd_origin`, which **fail closed** when their signal is absent. A malformed CIDR in a `source_ip` leaf is rejected with **422** `ROUTING_POLICY_INVALID`. The `query_shape` operand (#940) — `{"type": "query_shape", "any_of": ["JOIN", "SUBQUERY"]}` — matches a query that has any listed shape (`JOIN`, `UNION`, `SUBQUERY`, `CTE`, `GROUP_BY`, `HAVING`, `AGGREGATE`, `WINDOW_FUNCTION`) anywhere in the statement; an empty `any_of` is rejected with **422** `ROUTING_POLICY_INVALID`, and the leaf fails closed when the SQL cannot be parsed for its shape. Routing re-parses the stored SQL text with JSqlParser whatever the engine, so on a plugin datasource the leaf matches only when that text happens to be standard SQL JSqlParser can read (often true for a warehouse, never for a MongoDB or Redis command). Pair it with `ESCALATE` or `REQUIRE_APPROVALS` to send, say, every joined query to a second reviewer, or with `AUTO_REJECT` to refuse it outright; a grant's `denied_shapes` refuses at submission instead. The `estimated_bytes_scanned` operand (#941) — `{"type": "estimated_bytes_scanned", "operator": "GT", "value": 1000000000000}` — compares the warehouse's pre-flight bytes-scanned estimate (raw bytes, `value ≥ 0`) and, like `estimated_rows`, **fails closed**: it is `false` whenever no bytes estimate exists, which is every engine except BigQuery, Snowflake and Databricks. It is the advisory counterpart of the datasource and grant bytes-scanned cap. **Response 201:** Full routing-policy object (see the list shape below). `Location` header points to `/api/v1/admin/routing-policies/{id}`. **Response 400:** Bean Validation failure on the request body. `error: VALIDATION_ERROR`. @@ -4607,7 +4641,8 @@ follow up with one simulation per user of interest. It is deliberately not an N- "estimated_rows": null, "scan_type": null, "query_shapes": [], - "shapes_analyzed": true + "shapes_analyzed": true, + "estimated_bytes_scanned": null }, "caveats": ["CLIENT_CONTEXT_ABSENT", "COST_ESTIMATE_ABSENT"] } @@ -4637,11 +4672,12 @@ that did not apply is reported with `outcome: "SKIP"` rather than omitted. `outc | `SQL_PARSE` | `query_type`, `referenced_tables`, `transactional`, `has_where_clause`, `has_limit_clause`, `query_shapes` (#940 — the statement's shapes in declaration order), `shapes_analyzed` (`false` for every engine plugin — warehouses included, since only the in-process JSqlParser path reads shapes — and for a statement the walker cannot traverse) | whenever the statement parsed; `{}` when it did not | | `EFFECTIVE_PERMISSION` | `query_admin_short_circuit`, `contributing_grants[]`, `rejected_tables`, `denied_tables` (#939 — the referenced tables a `denied_schemas` / `denied_tables` entry reaches; checked after the allow-list, a non-empty list denies with `workflow.access_simulation.permission.table_denied`), `rejected_columns` (#935 — the denied entries the query reaches; a non-empty list denies with `workflow.access_simulation.permission.column_denied`), `denied_shapes` (#940 — the grant's denied shapes the query has, in declaration order; checked last, a non-empty list denies with `workflow.access_simulation.permission.shape_denied`), `expires_at` | always (`expires_at` omitted when the permission is standing or the caller is a `QUERY_ADMIN` holder) | | `SQL_REVIEW` | `blocking_rule_ids[]`, `blocking_count` | `MATCH` only — a deterministic SQL review rule fired at `BLOCK` (#864); `{}` on `NO_MATCH` | +| `BYTES_SCANNED_CAP` | `bytes_scanned_cap`, `bytes_scanned_cap_source`, `estimated_bytes_scanned`, `bytes_scanned_cap_outcome` | whenever a cap applies (#941); `{}` on `NO_MATCH` (no cap). In a simulation the step is always `SKIP` with no estimate — the cap is named, never compared. On a live trace: `ALLOW` within the cap, `DENY` over it or with no estimate under `REJECT` (the trace then ends `REJECTED` and every later decision stage is `SKIP`), `MATCH` with no estimate under `REQUIRE_REVIEW` — every auto-approve stage below then reports `bytes_cap_suppressed: true` | | `ROUTING_POLICIES` | `policies[]` | always (`[]` when the org has none) | -| | `matched_policy_id`, `matched_policy_name`, `action`, `effective_min_approvals`, `sql_review_suppressed` | `MATCH` only | +| | `matched_policy_id`, `matched_policy_name`, `action`, `effective_min_approvals`, `sql_review_suppressed`, `bytes_cap_suppressed` | `MATCH` only | | `GRANT_FAST_PATH` | `considered_grant_ids` | whenever grants were looked up | | | `grant_id`, `approver_email` | `MATCH`, and `NO_MATCH` when a covering grant was suppressed by a `BLOCK` finding | -| `REVIEW_PLAN` | `requires_human_approval`, `auto_approve_reads`, `sql_review_suppressed` | always (`sql_review_suppressed` absent on `SKIP`) | +| `REVIEW_PLAN` | `requires_human_approval`, `auto_approve_reads`, `sql_review_suppressed`, `bytes_cap_suppressed` | always (both `*_suppressed` keys absent on `SKIP`) | | | `review_plan_id`, `min_approvals_required` | only when the datasource has a review plan | | `ELIGIBLE_REVIEWERS` | `submitter_excluded` | always | | | `reviewers[]` (`user_id`, `email`, `display_name`) | when the datasource has its own reviewer assignment | @@ -4682,7 +4718,7 @@ endpoint reads both signals live: | `caveats` value | Meaning | |---|---| | `CLIENT_CONTEXT_ABSENT` | A hypothetical request carries no source IP, user agent or CI/CD origin, so every client-context routing condition (AF-446) evaluates to `false`. A policy keyed on one of those will report `matched: false` here and still fire on the real submission. | -| `COST_ESTIMATE_ABSENT` | The AF-624 pre-flight estimate is computed per submitted query, so `estimated_rows` / `scan_type` conditions evaluate to `false`. | +| `COST_ESTIMATE_ABSENT` | The AF-624 pre-flight estimate is computed per submitted query, so `estimated_rows` / `estimated_bytes_scanned` / `scan_type` conditions evaluate to `false`, and a bytes-scanned cap is reported (`BYTES_SCANNED_CAP` → `SKIP`) but not compared. | | `COLUMN_MATCH_BARE_NAME` | At least one masking policy's `column_ref` is a bare column name, which matches every table. See below. | | `ENGINE_CLASSIFICATION_UNAVAILABLE` | Row security could not be classified offline — the datasource's engine needs live schema knowledge (Cassandra / ScyllaDB). Reported as `UNKNOWN`, never as "no impact". | diff --git a/docs/05-backend.md b/docs/05-backend.md index 8335aec4e..337f67dea 100644 --- a/docs/05-backend.md +++ b/docs/05-backend.md @@ -1182,7 +1182,70 @@ The result is a `SelectExecutionResult` mapped to `SampleRowsResponse` for `GET - **Bounded.** Both calls run under the dedicated `accessflow.proxy.estimate-timeout` (`ACCESSFLOW_PROXY_ESTIMATE_TIMEOUT`, default `PT5S`) instead of the full execution statement timeout — an estimate is a best-effort signal, never worth a long lock. - **Every path persists a row.** Success stores the full estimate; engines with no plan concept store `supported=false` + a localized `unsupported_reason` (a transactional `BEGIN…COMMIT` envelope and a SQL Server dry-run under row security short-circuit the same way); an unexpected error stores a `failed=true` sentinel with the message — mirroring the AI module's sentinel convention, so the frontend can always render a definitive state. Completion publishes `QueryEstimateCompletedEvent` (or `QueryEstimateFailedEvent`), which the realtime module fans out as the `query.estimate_complete` WebSocket event. - **Row-security values are redacted from the plan.** The dry-run binds the submitter's resolved row-security values, and engines inline bound values into predicate text (PostgreSQL `Index Cond` / `Filter`, MySQL `attached_condition`, MongoDB stage filters). So whenever a row-security directive was resolved for the submitter, or the engine reports an applied policy, the service persists the plan tree with every node's `detail` set to null and `raw_plan` null (#1092). Operation, target, row and cost figures, and therefore routing and the AI prompt summary, are unaffected. Keeping predicate text safely is tracked in #1093 (generic-plan EXPLAIN) and #1094 (bound-value redaction). -- **Consumers.** `GET /queries/{id}` embeds the row as `cost_estimate`; `QueryReviewStateMachine.buildContext` reads it live (fail-closed) for the `estimated_rows` / `scan_type` routing conditions; `DefaultAiAnalyzerService` renders it into the `{{cost_estimate}}` prompt placeholder. +- **Warehouse bytes are persisted (#941).** The dry-run's `estimatedBytesScanned` (AF-634 — BigQuery, Snowflake, Databricks) is stored as `query_estimates.estimated_bytes_scanned`; it is null for every other engine. +- **Consumers.** `GET /queries/{id}` embeds the row as `cost_estimate`; `workflow.internal.routing.ConditionContextFactory` reads it live (fail-closed) for the `estimated_rows` / `estimated_bytes_scanned` / `scan_type` routing conditions; the bytes-scanned cap compares against it (see [Bytes-scanned cost caps](#bytes-scanned-cost-caps-941)); `DefaultAiAnalyzerService` renders it into the `{{cost_estimate}}` prompt placeholder, including a "will scan …" line when a bytes estimate exists. +- **No race on the AI-skipped path (#941).** When `ai_analysis_enabled = false`, nothing waits for the listener, so routing used to race it and an `estimated_rows` policy could silently fail closed on an estimate that was about to exist. `QueryReviewStateMachine.onAiSkipped` now calls `estimateSubmittedQuery(id)` before deciding — insert-once, so it returns the listener's row when that already landed and computes it otherwise — in a separate `REQUIRES_NEW` transaction, so losing the insert race to the listener never rolls back the decision (see [Bytes-scanned cost caps](#bytes-scanned-cost-caps-941)). The AI-failed path computes it only when a bytes-scanned cap applies. + +### Bytes-scanned cost caps (#941) + +On a bytes-billed warehouse, rows are the wrong unit: a query that returns ten rows can scan a +terabyte, and a row cap only truncates the result after the work is done and paid for. The +bytes-scanned cap refuses such a query **before** it runs, using the persisted pre-flight estimate. +It is not billing, chargeback or reconciliation against actual spend: estimates are estimates. + +- **Where it is set.** `datasources.max_bytes_scanned_per_query` (NULL = off) and a per-grant + `bytes_scanned_limit_override` on direct and group grants. `core.api.BytesScannedCapResolutionService` + (`DefaultBytesScannedCapResolutionService`) resolves the binding cap as the smaller of the + datasource cap and the user's merged grant override (smallest across grants, like + `row_limit_override`), naming the source — the datasource on a tie — and carrying the + datasource's `bytes_cap_missing_estimate` policy (`core.api.AppliedBytesCap`). +- **Only where an estimate can exist.** Both fields are refused at config time with 422 + `BYTES_SCANNED_CAP_NOT_SUPPORTED` on any engine outside `core.api.BytesScannedCapSupport` + (BigQuery, Snowflake, Databricks). On any other engine every query would lack an estimate, so a cap + there could only refuse everything or nothing. +- **Missing estimate is an explicit choice, never a silent pass.** A query can still lack a bytes + estimate on a supported engine (the estimate failed, DDL, a shape the engine cannot plan). The + datasource's `bytes_cap_missing_estimate` decides: `REQUIRE_REVIEW` (default) holds every automatic + approval for a person, `REJECT` refuses the query. +- **At the decision.** `QueryReviewStateMachine` resolves the cap at every entry point — together + with computing a missing estimate (AI skipped, or a cap applies) — in its own `REQUIRES_NEW` + transaction before the decision transaction reads the query. The estimate write is a + check-then-insert racing the estimate listener on a unique key, and it bumps the version of the + `query_requests` row; inside the decision transaction a lost race would roll back the transition + and strand the query in `PENDING_AI`. Isolated, a lost race only fails the inner transaction and + the winner's row is read. It then compares + the persisted estimate, stamps `query_requests.bytes_scanned_cap` / `_source` / `_outcome`, and + hands a `BytesCapCheck` to `QueryDecisionEvaluator` — the same explicit-input idiom as the SQL + review block. `EXCEEDED` and `NO_ESTIMATE_REJECTED` decide **first**, before routing and the + AI-failed path: `PENDING_AI → REJECTED` (`QueryDecisionKind.BYTES_CAP_REJECTED`, no + `routing_decision` row), published as `QueryAutoRejectedEvent` with a reason naming the estimate + and the cap, so the existing notification fan-out applies. `NO_ESTIMATE_REVIEW` joins the SQL + review guard: it suppresses a routing `AUTO_APPROVE`, the grant fast path and the plan's own + approvals, and never softens an `AUTO_REJECT`. The decision trace gains a `BYTES_SCANNED_CAP` step. +- **Again just before execution.** `DefaultQueryLifecycleService.doExecute` — the path of direct, + scheduled, recurring-occurrence and break-glass runs — re-resolves the cap and re-reads (or computes) + the estimate before anything else: a scheduled or recurring run executes long after its decision, + break-glass skips the decision entirely, and the cap may have been lowered since approval. A refusal + records `APPROVED → FAILED` with a localized `error_message` naming the estimate and the cap. + `GroupExecutionService` does the same per `QUERY` member, dry-running the member (only when a cap + applies, bounded by `accessflow.proxy.estimate-timeout` through + `QueryCostEstimateService.estimateBytesScanned`) since a member has no persisted estimate; a + refused member fails and `continue_on_error` decides the rest. At execution a missing estimate + under `REQUIRE_REVIEW` passes for a single query — the review it demands happened before approval + (break-glass relies on its mandatory retro-review; a scheduled or recurring query approved + *before* the cap was configured is the one gap, since nobody reviewed it against a cap). A group is + never checked against the cap when its plans auto-approve it, so a member with no estimate passes + only when a person approved the group (`error.bytes_cap.no_estimate_unreviewed` otherwise). + **Break-glass is capped**: a cost ceiling is a guardrail like the row cap, not a review step. +- **Audit.** One `QUERY_BYTES_SCANNED_CAP_ENFORCED` row (null actor, `trigger=bytes_scanned_cap`, + `stage=decision|execution`, `limit`, `source`, `outcome`, `estimated_bytes` when known) whenever the + cap changed the outcome; a successful `QUERY_EXECUTED` under a cap carries `bytes_scanned_cap`, + `bytes_scanned_cap_source` and `bytes_scanned_estimate`. +- **The advisory half.** The `estimated_bytes_scanned` routing condition escalates on the same + estimate without refusing anything, and fails closed when no estimate exists. +- **Simulation.** The access explainer has no submitted query and so no estimate: it names the cap + (`BYTES_SCANNED_CAP` → `SKIP`) and never compares it, under the `COST_ESTIMATE_ABSENT` caveat. +- No configuration knob: everything is per datasource and per grant. ### Data classification & derivation (AF-447) @@ -1625,6 +1688,8 @@ Decision rules: | `auto_approve_reads=true` AND `query_type=SELECT` AND AI risk ∈ {LOW, MEDIUM} | `APPROVED` (fast path) | | (default) | `PENDING_REVIEW` | | Datasource has no review plan | `PENDING_REVIEW` (safe default) | +| **Bytes-scanned cap refuses the query (#941)** — the estimate exceeds the cap, or there is none and the datasource says `REJECT`. Decided first, before routing and the AI-failed path | `REJECTED` — see [Bytes-scanned cost caps](#bytes-scanned-cost-caps-941) | +| **Bytes-scanned cap with no estimate under `REQUIRE_REVIEW` (#941)** | `PENDING_REVIEW` — suppresses the same three `APPROVED` rows as a SQL review `BLOCK` | | **Any `BLOCK` SQL review finding recorded at submission (#864)** — checked at every entry point, before all three rows above can approve | `PENDING_REVIEW` — each of the three `APPROVED` rows is suppressed (a routing `AUTO_APPROVE` too; `AUTO_REJECT` is untouched). Audited once as `SQL_REVIEW_BLOCKED` when it changed the outcome. See the "Submission enforcement (#864)" paragraph of [Deterministic SQL review rules](#deterministic-sql-review-rules-sqlreview-862) | `AiAnalysisFailedEvent` **always** transitions to `PENDING_REVIEW`, regardless of plan flags. Auto-approve is a positive-signal shortcut; failure is a missing signal — they aren't symmetric, so an AI provider error never short-circuits human review. The AI module persists a sentinel `CRITICAL` analysis row on failure with `failed=true` and `error_message=` (added in AF-249) so the reviewer can render an "AI analysis failed" surface on `QueryDetailPage` instead of seeing a fake CRITICAL verdict. Reviewers and admins can call [`POST /queries/{id}/reanalyze`](04-api-spec.md#post-queriesidreanalyze--response-202) to re-run analysis on the failed row — the workflow service deletes the sentinel and publishes `AiReanalysisRequestedEvent`, which the AI module's listener consumes by invoking the normal `analyzeSubmittedQuery` pipeline. A `QUERY_AI_REANALYZE_REQUESTED` audit row is written from the controller on each call. diff --git a/docs/06-frontend.md b/docs/06-frontend.md index 56b26f770..2f3c28806 100644 --- a/docs/06-frontend.md +++ b/docs/06-frontend.md @@ -321,7 +321,7 @@ Full detail view for any query: - When `status === 'TIMED_OUT'`, a warning callout above the SQL block names the review plan, the configured `approval_timeout_hours`, and how long ago the timeout fired. The metadata sidebar surfaces `plan` / `timeout.hours` for any query whose datasource has a review plan, regardless of status. Status-pill colour and label come from `statusColors.ts` (`TIMED_OUT` → warn-amber palette, label "TIMED OUT"). - When `ai_analysis.failed === true` (AF-249), a warning `Alert` at the top of the main column tells the reviewer that AI analysis didn't complete and that review is proceeding without an AI recommendation; the analyzer's reason is shown both in the banner detail and in a dedicated failure variant of the AI accordion. The `RiskPill` in the accordion header switches to a neutral grey "AI N/A" variant (`failed` prop on `RiskPill`). For `REVIEWER` / `ADMIN` callers a primary "Re-analyze" button (in both the banner and the accordion) calls `POST /queries/{id}/reanalyze`; the page invalidates its TanStack Query entries on success and picks up the new analysis via the existing `ai.analysis_complete` WebSocket event. The list page (`QueryListPage`) renders the same "AI N/A" pill in the risk column when `ai_failed=true` on the list row, so a CRITICAL-looking sentinel is never mistaken for a real risk verdict. - When the latest entry in `review_decisions[]` has `decision: REQUESTED_CHANGES` AND the query is still `PENDING_REVIEW` (AF-269), an info `Alert` at the top of the main column tells the submitter that the reviewer asked for changes — body interpolates `{{reviewer}}`, `{{when}}`, and `{{comment}}`. The reviewer decision panel itself requires a non-empty comment for both **Reject** (disabled until typed) and **Request changes** (already disabled); approving still allows an empty comment. The rejected stage of `ApprovalTimeline` carries the last `REJECTED` decision's comment (wrapped in `"…"` so the existing italic style in [ApprovalTimeline.tsx](../frontend/src/components/review/ApprovalTimeline.tsx) applies). -- **Cost-estimate card** (AF-624) — a "Cost estimate" `DetailCard` (`components/review/CostEstimatePanel.tsx`) renders the query's persisted pre-flight blast-radius estimate from `cost_estimate` on `GET /queries/{id}`: the exact affected-row count for UPDATE/DELETE ("Affected rows (exact)"), the plan's estimated rows / scan type / cost, and the execution-plan tree (reusing the editor's `PlanTree` + `utils/queryPlan.ts`), falling back to the raw plan text. State machine mirrors the AI card: while `status === 'PENDING_AI'` and `cost_estimate` is null it shows "Computing the cost estimate…"; a null estimate past that shows "No cost estimate is available for this query."; `supported=false` renders the localized `unsupported_reason` (still showing the exact count when one was computed); `failed=true` renders a warning with `error_message`. The `query.estimate_complete` WebSocket event invalidates `['queries','detail',id]` so the panel fills in without polling. +- **Cost-estimate card** (AF-624) — a "Cost estimate" `DetailCard` (`components/review/CostEstimatePanel.tsx`) renders the query's persisted pre-flight blast-radius estimate from `cost_estimate` on `GET /queries/{id}`: the exact affected-row count for UPDATE/DELETE ("Affected rows (exact)"), the plan's estimated rows / scan type / cost, the warehouse's **estimated bytes scanned** when the engine reports one (#941), and the execution-plan tree (reusing the editor's `PlanTree` + `utils/queryPlan.ts`), falling back to the raw plan text. State machine mirrors the AI card: while `status === 'PENDING_AI'` and `cost_estimate` is null it shows "Computing the cost estimate…"; a null estimate past that shows "No cost estimate is available for this query."; `supported=false` renders the localized `unsupported_reason` (still showing the exact count when one was computed); `failed=true` renders a warning with `error_message`. The `query.estimate_complete` WebSocket event invalidates `['queries','detail',id]` so the panel fills in without polling. - **SQL review findings card** (#865) — a "SQL review findings" `DetailCard` renders `sql_review_findings` from `GET /queries/{id}` through the shared `components/review/SqlReviewFindingList.tsx`, **only when at least one rule fired** (a clean query keeps its page short). When any finding is `BLOCK` a note above the list explains that the blocking rule(s) fired at submission, so the query could not auto-approve and was sent to human review — a BLOCK never rejects. Messages arrive localized from the backend; the frontend never formats a rule message. - **Approval-likelihood card** (AF-645) — rendered **only for viewers holding `QUERY_REVIEW` who are not the query's submitter** (`canSeeApprovalPrediction = isReviewer && submitterId !== user.id` — deliberately *not* the decision panel's `canDecide`, which additionally requires `PENDING_REVIEW`; the card stays useful after a decision lands): the prediction is a triage aid for whoever decides, and showing a submitter how their peers are likely to vote on their own open request would invite cancel-and-resubmit gaming. The backend applies the same predicate in `QueryReadController` and omits `approval_prediction` from the response entirely for those callers, so this gate is defence in depth rather than the only thing standing between a submitter and their own number. An "Approval likelihood" `DetailCard` (`components/review/ApprovalPredictionPanel.tsx`) renders the advisory approval-outcome prediction from `approval_prediction` on `GET /queries/{id}`: the label "Historical approval likelihood" next to the probability as a rounded percentage (`components/review/ApprovalPredictionBadge.tsx`), plus a permanent note — "Statistical estimate based on this organization's past review decisions. Advisory only — it never approves or rejects anything." States: with no row yet it shows "Computing the approval likelihood…" while `status === 'PENDING_AI'`, or while `PENDING_REVIEW` and the query's `updated_at` is inside a five-minute grace window — scoring fires off the transition into review and takes seconds, so past that the row is never coming (a query that predates the feature, or a listener that never ran) and the copy switches to "No approval likelihood is available for this query."; `failed=true` renders a warning; `skipped=true` — or any row that carries no probability — renders an info notice with the **client-localized** `skipped_reason` machine token (`DISABLED` → "…switched off for this organization.", `MODEL_NOT_SERVING` → "Not enough review history yet.", any other/unknown token → the generic unavailable copy, never the raw token; the token set is modelled as the `ApprovalPredictionSkipReason` union in `types/api.ts` so a new backend token breaks the lookup map at compile time). The badge deliberately uses the neutral `--fg-muted` / `--bg-sunken` / `--border` tokens rather than the `--risk-*` palette, and all copy is strictly non-directive — the number is a triage signal, never a recommendation to approve or reject. The `query.prediction_complete` WebSocket event invalidates `['queries','detail',id]` and `['reviews','pending']` so both the card and the queue column fill in without polling. - When `ai_analysis === null` and the query has already advanced out of `PENDING_AI` (AF-307), the AI step is rendered as **bypassed** rather than waiting. The card title becomes "AI analysis (skipped)" with a muted body — "AI analysis was skipped — this datasource has AI analysis disabled." — and the `ApprovalTimeline` shows a gray stage labeled "AI analysis skipped" (dot uses `--fg-muted`). The skipped state is derived on the frontend (`!ai_analysis && status !== 'PENDING_AI'`); the backend persists no `ai_analyses` row on the skip path. While the query is still in `PENDING_AI`, the original "Awaiting analysis…" fallback continues to render. @@ -333,7 +333,7 @@ Four-step flow at `/datasources/new` for adding a new datasource. Replaces a fla 1. **Type selection** — fetches `GET /datasources/types` and renders a grid of cards via `DatasourceTypeSelector`. Each card shows the logo (`icon_url`), display name, a one-line description, and a `DriverStatusBadge` (`READY` / `AVAILABLE` / `UNAVAILABLE`). Cards with `UNAVAILABLE` are disabled with a tooltip pointing the admin at the driver-cache configuration. Selecting a card advances to step 2 and seeds the form with `default_port` and `default_ssl_mode`. 2. **Connection details** — standard fields (name, host, port, database, username, password, ssl_mode), pre-filled from the type's defaults. A `JdbcUrlPreview` renders the URL live from `jdbc_url_template` as the user types. Bean-Validation errors surface inline. On first submit the wizard `POST`s `/datasources`; if the user returns to this step from **Test** and resubmits, the wizard `PUT`s the previously created record instead of issuing a second `POST` (which would 409 on the unique-name-per-org constraint). The primary button label switches from **Save and test** to **Save and continue** once a record exists. 3. **Test connection** — calls `POST /datasources/{id}/test` against the persisted datasource and surfaces latency or vendor error. **Back** returns to the connection step (record retained, resubmit becomes a `PUT`). **Next** advances to step 4 and is enabled only after a successful test; **Skip** lets admins proceed without a green test (e.g. air-gapped configuration). The first connection of a never-yet-resolved type may take 1–5 s due to driver download — show an explicit "Resolving driver…" state on the test button. -4. **Configuration** — review policy and limits before finishing: `connection_pool_size` (1–200), `max_rows_per_query` (1–1,000,000), `review_plan_id` (Select populated from `GET /review-plans`), `require_review_reads` / `require_review_writes` switches, the optional `environment` select (#865 — see DatasourceSettingsPage below), plus the AI analysis toggle and AI config selector (from `GET /ai-configs`). Submit issues `PUT /datasources/{id}` and navigates to `DatasourceSettingsPage` with a success toast. +4. **Configuration** — review policy and limits before finishing: `connection_pool_size` (1–200), `max_rows_per_query` (1–1,000,000), `review_plan_id` (Select populated from `GET /review-plans`), `require_review_reads` / `require_review_writes` switches, the optional `environment` select (#865 — see DatasourceSettingsPage below), plus the AI analysis toggle and AI config selector (from `GET /ai-configs`). On a BigQuery, Snowflake or Databricks datasource (`supportsBytesCap` in `src/utils/bytesCap.ts`) the step also offers the optional **Max bytes scanned per query** (`BytesInput`, rule mirroring the backend `@Min(1)`) and the **When no bytes estimate is available** select (`REQUIRE_REVIEW` / `REJECT`, #941); neither is ever sent for another engine. Submit issues `PUT /datasources/{id}` and navigates to `DatasourceSettingsPage` with a success toast. The wizard is the only entry point that materializes a datasource; `DatasourceListPage` links to it via a "New datasource" button. @@ -380,10 +380,11 @@ immediately. ### DatasourceSettingsPage *(ADMIN)* - Connection config form with live test button (`POST /datasources/{id}/test`) +- **Bytes-scanned cap** (#941) — on BigQuery, Snowflake and Databricks only (`supportsBytesCap`), the limits section adds **Max bytes scanned per query** (`BytesInput`, optional, `@Min(1)` parity) and **When no bytes estimate is available** (`REQUIRE_REVIEW` / `REJECT`). Emptying a stored cap sends `clear_max_bytes_scanned_per_query: true` (`toBytesCapUpdate`), since a null cap means "unchanged" to the API - **Schema** tab (AF-443) — a searchable, hierarchical object tree (`components/datasources/SchemaObjectTree.tsx`) over the introspected schema: schemas → tables → columns, with a single filter that matches across **all three levels** (a column-name query surfaces its table and schema). The pure filter logic lives in `src/utils/schemaFilter.ts` (`filterSchema`). Each table row has a **"Preview data"** action that opens `components/datasources/SampleDataDrawer.tsx` → `SampleDataPreview.tsx`, a read-only AntD `Table` of a bounded, **RLS- and masking-aware** sample fetched via `useTableSample` (`GET /datasources/{id}/sample-rows`). Masked columns are badged with a lock icon and only ever render the masked value; a banner notes that row-level security and masking are applied, and the footer shows the row count / cap. The same `SchemaObjectTree` + `SampleDataDrawer` power the editor sidebar (`components/editor/SchemaTree.tsx`), so the cross-hierarchy search and sample preview are available while writing queries too. - **ER diagram** tab (`components/datasources/ErDiagramTab.tsx` → `ErDiagram.tsx`) — renders the introspected schema as a `@xyflow/react` graph, one node per table (showing columns + PK markers) and one edge per foreign key (label `from → to`). Auto-layout via `dagre` (LR rank direction); read-only — `nodesDraggable={false}`. Clicking a node highlights all edges touching it (others fade to opacity 0.18); clicking the canvas background clears the selection. Loading state is a same-size `Skeleton.Node` to avoid CLS; databases without FKs (denormalized warehouses, custom drivers without `getImportedKeys`) render an `EmptyState`. The CSS in `src/styles/globals.css` already honours `prefers-reduced-motion` for all transitions. - `PermissionMatrix` — table of all users × (can_read, can_write, can_ddl, can_break_glass, row_limit, allowed_schemas, restricted columns count, denied columns count, denied tables count, denied shapes count, expires_at). Restricted columns render as `"N columns"` with a hover tooltip listing the fully-qualified names; `"—"` when none. The break-glass column uses the shared `PermCell` check/dash renderer (colour + icon, never colour alone). -- `GrantAccessModal` includes a `can_break_glass` switch (after the read/write/DDL trio, default off, AF-385) and a `restricted_columns` multi-select populated from the datasource's introspected schema (`flattenSchemaToColumns` in `src/utils/schemaColumns.ts`). The break-glass help text explains it is an **additional** emergency capability that bypasses review but still requires the underlying read/write/DDL capability at submission time; the existing "at least one of read/write/DDL" rule is preserved and is **not** satisfied by break-glass alone. The restricted-columns help text explains that values are masked in results and the AI reviewer is informed but does not auto-reject. A `denied_columns` multi-select (#935), fed by the same column options, is shown only for the JSqlParser engines (`supportsDeniedColumns` in `src/utils/deniedColumns.ts`, which reuses `SQL_REVIEW_DB_TYPES`). Its `Form.Item` validator mirrors the backend `@Pattern`/`@Size`: each entry must be `table.column` or `schema.table.column`, at most 200 entries. Both the user and the group permission tables gain a "Denied columns" column: a red "N columns" tag with the columns in a tooltip, or "—". Two `mode="tags"` selects, **Denied schemas** and **Denied tables** (#939), follow; their options come from the introspected schema, and table options are schema-qualified (`schema.table`, narrowed to the selected allowed schemas when any are chosen) because a bare name would deny that table in every schema. Free-typed entries are accepted. Validators in `src/utils/deniedTables.ts` mirror the backend `@Size`/`@NotBlank` (at most 50 schemas and 200 tables, no blank entry). Both permission tables render a "Denied tables" column: a red count tag whose tooltip lists the denied schemas as `schema.*` and then the denied tables (`deniedTableEntries`), or "—". A **Denied query shapes** multi-select (#940) follows, offering the eight `QueryShape` values labelled through `queryShapeLabel` (`enums.query_shape.*`); like denied columns it is shown only for the JSqlParser engines (`supportsDeniedShapes` in `src/utils/deniedShapes.ts`), and its `Form.Item` rule mirrors the backend `@Size(max = 8)`. An empty selection is sent as `null`. Both permission tables render a "Denied shapes" column: a red "N shapes" tag whose tooltip lists the labelled shapes, or "—". Permissions are create-only (revoke + re-grant); there is no edit flow. +- `GrantAccessModal` includes a `can_break_glass` switch (after the read/write/DDL trio, default off, AF-385) and a `restricted_columns` multi-select populated from the datasource's introspected schema (`flattenSchemaToColumns` in `src/utils/schemaColumns.ts`). The break-glass help text explains it is an **additional** emergency capability that bypasses review but still requires the underlying read/write/DDL capability at submission time; the existing "at least one of read/write/DDL" rule is preserved and is **not** satisfied by break-glass alone. The restricted-columns help text explains that values are masked in results and the AI reviewer is informed but does not auto-reject. A `denied_columns` multi-select (#935), fed by the same column options, is shown only for the JSqlParser engines (`supportsDeniedColumns` in `src/utils/deniedColumns.ts`, which reuses `SQL_REVIEW_DB_TYPES`). Its `Form.Item` validator mirrors the backend `@Pattern`/`@Size`: each entry must be `table.column` or `schema.table.column`, at most 200 entries. Both the user and the group permission tables gain a "Denied columns" column: a red "N columns" tag with the columns in a tooltip, or "—". Two `mode="tags"` selects, **Denied schemas** and **Denied tables** (#939), follow; their options come from the introspected schema, and table options are schema-qualified (`schema.table`, narrowed to the selected allowed schemas when any are chosen) because a bare name would deny that table in every schema. Free-typed entries are accepted. Validators in `src/utils/deniedTables.ts` mirror the backend `@Size`/`@NotBlank` (at most 50 schemas and 200 tables, no blank entry). Both permission tables render a "Denied tables" column: a red count tag whose tooltip lists the denied schemas as `schema.*` and then the denied tables (`deniedTableEntries`), or "—". A **Denied query shapes** multi-select (#940) follows, offering the eight `QueryShape` values labelled through `queryShapeLabel` (`enums.query_shape.*`); like denied columns it is shown only for the JSqlParser engines (`supportsDeniedShapes` in `src/utils/deniedShapes.ts`), and its `Form.Item` rule mirrors the backend `@Size(max = 8)`. An empty selection is sent as `null`. Both permission tables render a "Denied shapes" column: a red "N shapes" tag whose tooltip lists the labelled shapes, or "—". On a BigQuery, Snowflake or Databricks datasource (`supportsBytesCap`) the modal also offers a **Bytes-scanned cap** (`bytes_scanned_limit_override`, `BytesInput`, rule mirroring `@Min(1)`, #941), and both permission tables gain a "Bytes cap" column; neither appears for another engine, where the backend would answer 422 `BYTES_SCANNED_CAP_NOT_SUPPORTED`. Permissions are create-only (revoke + re-grant); there is no edit flow. - **Masking** tab (`components/datasources/MaskingTab.tsx`, AF-381) — a table of dynamic data masking policies (column, strategy, reveal-to summary, enabled) with a create/edit modal. The modal picks a column via an `AutoComplete` from the introspected schema, a strategy `Select` driven by `enumOptions(MASKING_STRATEGIES, maskingStrategyLabel, t)`, a conditional `visible_suffix` field shown only for `PARTIAL`, and reveal-to multi-selects for roles (`enumOptions`), groups, and users. A **live preview** renders the masked output of an editable sample value through `src/utils/maskingPreview.ts` (a pure client-side mirror of the backend `ColumnMasker` strategies; `HASH` shows an illustrative fixed digest since the real SHA-256 is computed server-side). CRUD calls `src/api/maskingPolicies.ts`; validation parity matches the backend DTO (required column ≤ 512 chars, required strategy, `visible_suffix` 1–256). - **Row security** tab (`components/datasources/RowSecurityTab.tsx`, AF-380) — a table of row-level security policies (table, `column operator value` predicate, applies-to summary, enabled) with a create/edit modal. The structured form picks a table and column via `AutoComplete`s from the introspected schema, an operator `Select` (`enumOptions(ROW_SECURITY_OPERATORS, rowSecurityOperatorLabel, t)`), a value-source `Select` (`VARIABLE` | `LITERAL`), and a value field that switches to a `:user.*` variable `AutoComplete` (offering the `:user.id` / `:user.email` / `:user.role` / `:user.groups` built-ins) when the source is `VARIABLE`. Applies-to multi-selects target roles, groups, and users (empty = everyone). CRUD calls `src/api/rowSecurityPolicies.ts`; validation parity matches the backend DTO (required table/column/value ≤ 512 chars, required operator, required value source). - **Row limits** tab (`components/datasources/RowLimitTab.tsx`, #934) — a table of per-table row-limit policies (`schema.table`, max rows, applies-to summary, enabled) with a create/edit modal. The modal picks an optional schema and a table through `AutoComplete`s fed by the introspected schema (the table list narrows to the chosen schema; an empty schema means "any schema"), a `max_rows` `InputNumber` (1–1,000,000) and applies-to multi-selects for roles, groups and users (empty = everyone, admins included). CRUD calls `src/api/rowLimitPolicies.ts`; validation parity matches the backend DTO (required table ≤ 255 chars, optional schema ≤ 255 chars, required `max_rows` 1–1,000,000). @@ -1473,8 +1474,15 @@ summary lists the labelled shapes, and the decision trace labels `query_shapes` details the same way. The builder covers the `estimated_rows` (comparison operator + row count) and `scan_type` (glob tags, e.g. `Seq*`, `COLLSCAN`) pre-flight-estimate operands (AF-624) alongside the -original leaf set. `QueryDetailPage` shows a **matched-policy** alert when `GET /queries/{id}` -returns a non-null `matched_policy`. +original leaf set, plus `estimated_bytes_scanned` (#941): a comparison operator and a byte count +entered through the shared `components/common/BytesInput.tsx` (amount + MB/GB/TB/PB unit select, +decimal units; the form value and the wire value are always raw bytes — helpers `pickUnit` / +`byteUnitFactor` in `src/utils/bytesCap.ts`). Its summary renders through `formatBytes`. `QueryDetailPage` shows a **matched-policy** alert when `GET /queries/{id}` +returns a non-null `matched_policy`, and a **bytes-scanned cap** alert (`data-testid="bytes-cap-banner"`, +#941) when it returns `bytes_scanned_cap`: the outcome label (`enums.bytes_scanned_cap_outcome.*`) as +the title — an error for `EXCEEDED` / `NO_ESTIMATE_REJECTED`, a warning for `NO_ESTIMATE_REVIEW`, info +for `WITHIN` — and the estimate, the cap and its source in the body. The decision trace labels the +`BYTES_SCANNED_CAP` step and formats its byte details through `formatBytes`. ### Policy simulator (AF-630) diff --git a/docs/07-security.md b/docs/07-security.md index 4b41992ac..620572cdb 100644 --- a/docs/07-security.md +++ b/docs/07-security.md @@ -1017,6 +1017,31 @@ still admits a single-table `UPDATE … WHERE` — so pair them with the read/wr or auto-rejects a shape through a routing policy instead of refusing it at the grant. It fails closed the other way — an unanalysed shape does not match — so prefer the grant when the rule must hold. +### Bytes-scanned cost caps (#941) + +A pre-execution **cost guardrail** for bytes-billed warehouses (BigQuery, Snowflake, Databricks), +where a row cap does nothing about cost: `datasources.max_bytes_scanned_per_query` and a per-grant +`bytes_scanned_limit_override` refuse a query whose pre-flight bytes-scanned estimate exceeds the +cap. It is a resource guardrail on an **estimate**, not an access boundary and not a billing control: +an over-estimate refuses a legitimate query, an under-estimate lets an expensive one through. + +- **Merge — most restrictive wins.** The datasource cap and every contributing grant's override + combine to the smallest value, so a permissive group grant can never raise a tight cap, and a JIT + approval that replaces a time-boxed row carries the replaced row's override over — an approval + never widens cost. +- **Configurable only where it can hold.** The engines that report no bytes estimate refuse the + fields at config time (422 `BYTES_SCANNED_CAP_NOT_SUPPORTED`); a cap there would silently never + fire, or refuse everything. +- **Fails closed, explicitly.** A capped query with no estimate is never silently allowed: the + datasource's `bytes_cap_missing_estimate` either holds every automatic approval for a person + (`REQUIRE_REVIEW`, the default) or refuses the query (`REJECT`). +- **Enforced twice, for everyone.** When the query leaves `PENDING_AI` (rejected before routing, so + no policy and no plan can approve it) and again just before execution — which covers scheduled, + recurring, grouped and **break-glass** runs and a cap lowered after approval. `QUERY_ADMIN` holders + are not exempt: the cap binds the datasource, not the reviewer. +- **Audited.** `QUERY_BYTES_SCANNED_CAP_ENFORCED` (null actor) records the limit, its source, the + estimate and the outcome whenever the cap changed an outcome. + ### Dynamic data masking policies (AF-381) `masking_policy` rows extend the static masking above with **per-column strategies** and a diff --git a/e2e/tests/admin-routing-policies.spec.ts b/e2e/tests/admin-routing-policies.spec.ts index ce7847f2b..ae2adf352 100644 --- a/e2e/tests/admin-routing-policies.spec.ts +++ b/e2e/tests/admin-routing-policies.spec.ts @@ -19,6 +19,8 @@ const BUILDER_POLICY_NAME = `Builder policy ${UNIQUE_SUFFIX}`; const AUTO_REJECT_POLICY_NAME = `Auto-reject deletes ${UNIQUE_SUFFIX}`; const CICD_REJECT_POLICY_NAME = `Block CI/CD ${UNIQUE_SUFFIX}`; const JOIN_REJECT_POLICY_NAME = `Block joins ${UNIQUE_SUFFIX}`; +const BYTES_BUILDER_POLICY_NAME = `Escalate big scans ${UNIQUE_SUFFIX}`; +const BYTES_REJECT_POLICY_NAME = `Block big scans ${UNIQUE_SUFFIX}`; const ROUTED_DS_NAME = `Routed DS ${UNIQUE_SUFFIX}`; const DEFAULT_API_BASE = 'http://localhost:8080'; @@ -205,4 +207,86 @@ test.describe.serial('/admin/routing-policies — routing engine', () => { ) .not.toMatch(/^(PENDING_AI|REJECTED)$/); }); + + // #941 — the builder offers the estimated_bytes_scanned operand and sends raw bytes. + test('builds an estimated-bytes-scanned condition entered in TB', async ({ page }) => { + await login(page, ADMIN_EMAIL, ADMIN_PASSWORD); + await page.goto('/admin/routing-policies'); + await waitForRoutingPoliciesListReady(page); + + await page.getByRole('button', { name: 'Add policy' }).first().click(); + const modal = page.getByRole('dialog').filter({ hasText: 'Add routing policy' }).first(); + await expect(modal).toBeVisible({ timeout: 10_000 }); + await modal.getByLabel('Name').fill(BYTES_BUILDER_POLICY_NAME); + await modal.getByLabel('Priority').fill(String(100_000 + Math.floor(Math.random() * 800_000))); + + // The seeded row is "Query type"; switch its operand. + await modal.locator('.ant-select').filter({ hasText: 'Query type' }).first().click(); + // The operand list is virtualised — off-screen options are not in the DOM — so walk it with + // the keyboard until the active option is the one we want. + const active = page.locator('.ant-select-item-option-active'); + for (let i = 0; i < 30; i += 1) { + if ((await active.textContent()) === 'Estimated bytes scanned') break; + await page.keyboard.press('ArrowDown'); + } + await expect(active).toHaveText('Estimated bytes scanned'); + await page.keyboard.press('Enter'); + const bytes = modal.getByLabel('Bytes scanned'); + await expect(bytes).toHaveValue('1'); + await bytes.fill('2'); + + const createResponse = page.waitForResponse( + (r) => + r.request().method() === 'POST' && + /\/api\/v1\/admin\/routing-policies$/.test(r.url()), + { timeout: 15_000 }, + ); + await modal.getByRole('button', { name: 'Create policy' }).click(); + const response = await createResponse; + expect(response.status()).toBe(201); + const created = (await response.json()) as CreatedRoutingPolicy & { + condition: { children?: { type: string; operator: string; value: number }[] }; + }; + createdPolicyIds.push(created.id); + expect(created.condition.children?.[0]).toEqual({ + type: 'estimated_bytes_scanned', + operator: 'GT', + value: 2_000_000_000_000, + }); + }); + + // #941 — PostgreSQL reports no bytes estimate, so the condition fails closed and never fires. + test('an estimated_bytes_scanned condition never fires without a bytes estimate', async ({ + request, + }) => { + const policy = await createRoutingPolicyViaApi(request, adminAccessToken, { + name: BYTES_REJECT_POLICY_NAME, + datasource_id: datasourceId as string, + priority: 100_000 + Math.floor(Math.random() * 800_000), + enabled: true, + action: 'AUTO_REJECT', + reason: 'scans are too large', + condition: { type: 'estimated_bytes_scanned', operator: 'GTE', value: 0 }, + }); + createdPolicyIds.push(policy.id); + + const query = await submitQueryViaApi( + request, + adminAccessToken, + datasourceId as string, + 'SELECT id FROM accounts WHERE id = 2', + 'e2e: bytes-scanned routing', + ); + await expect + .poll( + async () => { + const res = await request.get(`${apiBase()}/api/v1/queries/${query.id}`, { + headers: { Authorization: `Bearer ${adminAccessToken}` }, + }); + return ((await res.json()) as { status: string }).status; + }, + { timeout: 20_000 }, + ) + .not.toMatch(/^(PENDING_AI|REJECTED)$/); + }); }); diff --git a/frontend/src/components/common/BytesInput.tsx b/frontend/src/components/common/BytesInput.tsx new file mode 100644 index 000000000..508eb3470 --- /dev/null +++ b/frontend/src/components/common/BytesInput.tsx @@ -0,0 +1,63 @@ +import { useState } from 'react'; +import { InputNumber, Select, Space } from 'antd'; +import { useTranslation } from 'react-i18next'; +import { BYTE_INPUT_UNITS, byteUnitFactor, pickUnit, type ByteInputUnit } from '@/utils/bytesCap'; + +interface BytesInputProps { + /** Raw bytes; the form stores and submits bytes, the unit is display-only. */ + value?: number | null; + onChange?: (value: number | null) => void; + id?: string; + disabled?: boolean; + placeholder?: string; + 'aria-label'?: string; +} + +/** + * A byte count entered as an amount plus a unit (#941). Controlled through `value`/`onChange` so it + * drops into a `Form.Item` whose validation rules apply to the raw byte value. + */ +export function BytesInput({ + value, + onChange, + id, + disabled, + placeholder, + 'aria-label': ariaLabel, +}: BytesInputProps) { + const { t } = useTranslation(); + const [unit, setUnit] = useState(() => pickUnit(value)); + const factor = byteUnitFactor(unit); + const amount = value === null || value === undefined ? null : value / factor; + + const emit = (nextAmount: number | null, nextFactor: number) => { + onChange?.(nextAmount === null ? null : Math.round(nextAmount * nextFactor)); + }; + + return ( + + + id={id} + min={0} + step={1} + disabled={disabled} + placeholder={placeholder} + aria-label={ariaLabel} + value={amount} + onChange={(next) => emit(next, factor)} + style={{ width: '100%' }} + /> + + value={unit} + disabled={disabled} + aria-label={t('common.bytes_unit')} + style={{ width: 90 }} + options={BYTE_INPUT_UNITS.map((u) => ({ value: u.unit, label: u.unit }))} + onChange={(next) => { + setUnit(next); + emit(amount, byteUnitFactor(next)); + }} + /> + + ); +} diff --git a/frontend/src/components/common/__tests__/BytesInput.test.tsx b/frontend/src/components/common/__tests__/BytesInput.test.tsx new file mode 100644 index 000000000..b856e5da9 --- /dev/null +++ b/frontend/src/components/common/__tests__/BytesInput.test.tsx @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from 'vitest'; +import { fireEvent, render, screen } from '@testing-library/react'; +import '@/i18n'; +import { BytesInput } from '../BytesInput'; + +describe('BytesInput', () => { + it('shows a byte value in the largest unit it fills', () => { + render(); + expect(screen.getByLabelText('cap')).toHaveValue('2'); + expect(screen.getByText('TB')).toBeInTheDocument(); + }); + + it('emits raw bytes for the typed amount', () => { + const onChange = vi.fn(); + render(); + + fireEvent.change(screen.getByLabelText('cap'), { target: { value: '3' } }); + + expect(onChange).toHaveBeenLastCalledWith(3_000_000_000); + }); + + it('emits null when the amount is cleared', () => { + const onChange = vi.fn(); + render(); + + fireEvent.change(screen.getByLabelText('cap'), { target: { value: '' } }); + + expect(onChange).toHaveBeenLastCalledWith(null); + }); + + it('rescales the value when the unit changes', () => { + const onChange = vi.fn(); + render(); + + fireEvent.mouseDown(screen.getByRole('combobox')); + fireEvent.click(screen.getByTitle('TB')); + + expect(onChange).toHaveBeenLastCalledWith(2_000_000_000_000); + }); +}); diff --git a/frontend/src/components/policies/decisionTraceDetails.test.ts b/frontend/src/components/policies/decisionTraceDetails.test.ts index 84378f4fa..0e4bbcf94 100644 --- a/frontend/src/components/policies/decisionTraceDetails.test.ts +++ b/frontend/src/components/policies/decisionTraceDetails.test.ts @@ -186,4 +186,23 @@ describe('formatStepDetails — masking, omission and enum values (#1066 review) expect(byKey.scope).toBe('PIPELINE'); expect(rows.find((r) => r.key === 'approver_email')?.label).toBe('Approver'); }); + + it('formats the bytes-scanned cap step with byte sizes and labelled enums (#941)', () => { + const rows = formatStepDetails( + { + bytes_scanned_cap: 1_000_000_000_000, + bytes_scanned_cap_source: 'GRANT', + estimated_bytes_scanned: 2_500_000_000, + bytes_scanned_cap_outcome: 'EXCEEDED', + }, + t, + ); + const byKey = Object.fromEntries(rows.map((r) => [r.key, r.value])); + expect(byKey.bytes_scanned_cap).toBe('1 TB'); + expect(byKey.estimated_bytes_scanned).toBe('2.5 GB'); + expect(byKey.bytes_scanned_cap_source).toBe('Grant override'); + expect(byKey.bytes_scanned_cap_outcome).toBe( + 'Rejected: the estimate exceeds the bytes-scanned cap', + ); + }); }); diff --git a/frontend/src/components/policies/decisionTraceDetails.ts b/frontend/src/components/policies/decisionTraceDetails.ts index 2dc1294cd..55cb70050 100644 --- a/frontend/src/components/policies/decisionTraceDetails.ts +++ b/frontend/src/components/policies/decisionTraceDetails.ts @@ -1,5 +1,7 @@ import type { TFunction } from 'i18next'; import type { + BytesScannedCapOutcome, + BytesScannedCapSource, MaskingStrategy, QueryShape, QueryStatus, @@ -9,12 +11,17 @@ import type { RoutingPolicyTraceEntry, } from '@/types/api'; import { fmtDate } from '@/utils/dateFormat'; +import { formatBytes } from '@/utils/queryPlan'; import { + BYTES_SCANNED_CAP_OUTCOMES, + BYTES_SCANNED_CAP_SOURCES, MASKING_STRATEGIES, QUERY_SHAPES, QUERY_TYPES, RISK_LEVELS, ROUTING_ACTIONS, + bytesScannedCapOutcomeLabel, + bytesScannedCapSourceLabel, maskingStrategyLabel, queryShapeLabel, queryStatusLabel, @@ -47,6 +54,10 @@ export const KNOWN_DETAIL_KEYS = [ 'behavior', 'blocking_count', 'blocking_rule_ids', + 'bytes_cap_suppressed', + 'bytes_scanned_cap', + 'bytes_scanned_cap_outcome', + 'bytes_scanned_cap_source', 'can_break_glass', 'can_read', 'can_trigger', @@ -65,6 +76,7 @@ export const KNOWN_DETAIL_KEYS = [ 'environment_allows_break_glass', 'environment_name', 'environment_required_approvals', + 'estimated_bytes_scanned', 'estimated_rows', 'evaluated_at', 'expires_at', @@ -152,6 +164,17 @@ function enumValue(key: string, value: unknown, t: TFunction): string | null { } if (key === 'status' && includes(QUERY_STATUSES, value)) return queryStatusLabel(t, value); if (key === 'risk_level' && includes(RISK_LEVELS, value)) return riskLevelLabel(t, value); + if (key === 'bytes_scanned_cap_source' + && includes(BYTES_SCANNED_CAP_SOURCES, value)) { + return bytesScannedCapSourceLabel(t, value); + } + if (key === 'bytes_scanned_cap_outcome' + && includes(BYTES_SCANNED_CAP_OUTCOMES, value)) { + return bytesScannedCapOutcomeLabel(t, value); + } + if ((key === 'bytes_scanned_cap' || key === 'estimated_bytes_scanned') && typeof value === 'number') { + return formatBytes(value); + } return null; } diff --git a/frontend/src/components/review/CostEstimatePanel.test.tsx b/frontend/src/components/review/CostEstimatePanel.test.tsx index 6dca0eae4..5d2c51182 100644 --- a/frontend/src/components/review/CostEstimatePanel.test.tsx +++ b/frontend/src/components/review/CostEstimatePanel.test.tsx @@ -67,6 +67,22 @@ describe('CostEstimatePanel', () => { expect(screen.getByText('44,543.5')).toBeInTheDocument(); }); + it('renders the warehouse bytes-scanned estimate when present (#941)', () => { + render( + , + ); + expect(screen.getByText('Estimated bytes scanned')).toBeInTheDocument(); + expect(screen.getByText('1.5 TB')).toBeInTheDocument(); + }); + + it('omits the bytes line for engines that report none', () => { + render(); + expect(screen.queryByText('Estimated bytes scanned')).not.toBeInTheDocument(); + }); + it('renders the exact affected-row count for writes when present', () => { render( + {estimate.estimated_bytes_scanned !== null && + estimate.estimated_bytes_scanned !== undefined && ( +

+ {t('cost_estimate_panel.est_bytes_label')} + {formatBytes(estimate.estimated_bytes_scanned)} +
+ )} {estimate.scan_type && (
{t('cost_estimate_panel.scan_type_label')} diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.json index 4282e3447..e7278c1ab 100644 --- a/frontend/src/locales/de.json +++ b/frontend/src/locales/de.json @@ -36,7 +36,8 @@ "view_docs_aria": "Dokumentation anzeigen (wird in einem neuen Tab geöffnet)", "on_behalf_of": "im Auftrag von {{name}}", "on_behalf_of_tooltip": "Ein API-Key-Aufrufer hat für diese Person gehandelt (X-AccessFlow-On-Behalf-Of). Die genannte Person gilt für das Selbstgenehmigungsverbot als Einreicher.", - "principal_service_account_tooltip": "Eine nicht-menschliche Identität: sie authentifiziert sich nur per API-Key und kann sich nie interaktiv anmelden." + "principal_service_account_tooltip": "Eine nicht-menschliche Identität: sie authentifiziert sich nur per API-Key und kann sich nie interaktiv anmelden.", + "bytes_unit": "Einheit" }, "nav": { "editor": "SQL-Editor", @@ -654,7 +655,8 @@ "affected_rows_label": "Betroffene Zeilen (exakt)", "est_rows_label": "Geschätzte Zeilen", "scan_type_label": "Scan-Typ", - "cost_label": "Geschätzte Kosten" + "cost_label": "Geschätzte Kosten", + "est_bytes_label": "Geschätzte gescannte Bytes" }, "approval_prediction": { "value_label": "Historische Genehmigungswahrscheinlichkeit", @@ -864,7 +866,9 @@ "hint": "Effektives SQL ist die Anweisung, wie sie tatsächlich ausgeführt wurde, mit angewendeten Zeilensicherheits-Filtern und Soft-Delete-Umschreibungen. Gebundene Werte werden als ? angezeigt und nie gespeichert.", "diff_submitted_label": "Eingereichtes SQL", "diff_effective_label": "Effektives SQL" - } + }, + "bytes_cap_body": "Geschätzter Scan: {{estimate}} · Limit: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "keine Schätzung" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Umgebung", "environment_not_set": "Nicht gesetzt (Standardregeln der Organisation)", - "environment_help": "Legt fest, welcher SQL-Prüfregelsatz für Abfragen an dieser Datenquelle gilt." + "environment_help": "Legt fest, welcher SQL-Prüfregelsatz für Abfragen an dieser Datenquelle gilt.", + "label_max_bytes_scanned": "Max. gescannte Bytes pro Abfrage", + "max_bytes_scanned_help": "Abfragen ablehnen, deren Scan-Schätzung vor der Ausführung größer ist. Leer lassen für kein Limit.", + "max_bytes_scanned_placeholder": "Kein Limit", + "label_bytes_cap_missing_estimate": "Wenn keine Byte-Schätzung vorliegt", + "bytes_cap_missing_estimate_help": "Gilt nur, solange ein Limit gesetzt ist. „Menschliche Prüfung verlangen“ hält automatische Genehmigungen für eine Person zurück; „Ablehnen“ weist die Abfrage ab.", + "perm_col_bytes_cap": "Byte-Limit", + "perm_bytes_cap_none": "Standard", + "grant_bytes_cap_label": "Limit für gescannte Bytes", + "grant_bytes_cap_help": "Ein niedrigeres Limit für diesen Grant. Es gilt das strengste aus Datenquellen-Limit und allen Grants.", + "grant_bytes_cap_min": "Das Limit für gescannte Bytes muss mindestens 1 Byte betragen.", + "grant_bytes_cap_placeholder": "Standard der Datenquelle" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "z. B. *curl*, *GitHubActions*", "minutes_suffix": "Min.", "cicd_present_label": "Ist CI/CD-Herkunft", - "scan_type_placeholder": "z. B. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "z. B. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Gescannte Bytes" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Zeit seit letzter Genehmigung", "cicd_origin": "CI/CD-Herkunft", "estimated_rows": "Geschätzte Zeilen", - "scan_type": "Scan-Typ" + "scan_type": "Scan-Typ", + "estimated_bytes_scanned": "Geschätzte gescannte Bytes" }, "query_shape": { "JOIN": "Join", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Berechtigte Prüfer", "ROW_SECURITY": "Zeilensicherheit", "MASKING": "Maskierung", - "BREAK_GLASS": "Break-Glass" + "BREAK_GLASS": "Break-Glass", + "BYTES_SCANNED_CAP": "Limit für gescannte Bytes" }, "api_decision_step": { "CONNECTOR_GATES": "Konnektor-Prüfungen", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Feste Rate", "CRON": "Cron", "ONE_TIME": "Einmalig" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Menschliche Prüfung verlangen", + "REJECT": "Abfrage ablehnen" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Datenquellen-Limit", + "GRANT": "Grant-Override" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Innerhalb des Limits für gescannte Bytes", + "EXCEEDED": "Abgelehnt: die Schätzung überschreitet das Limit für gescannte Bytes", + "NO_ESTIMATE_REVIEW": "Zur Prüfung zurückgehalten: keine Byte-Schätzung unter dem Limit", + "NO_ESTIMATE_REJECTED": "Abgelehnt: keine Byte-Schätzung unter dem Limit" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "User-Agent des Anfragenden", "risk_level": "Risikostufe", "risk_score": "Risikowert", - "scan_type": "Scan-Typ" + "scan_type": "Scan-Typ", + "bytes_cap_suppressed": "Durch das Byte-Limit unterdrückt", + "bytes_scanned_cap": "Limit für gescannte Bytes", + "bytes_scanned_cap_outcome": "Ergebnis des Limits", + "bytes_scanned_cap_source": "Quelle des Limits", + "estimated_bytes_scanned": "Geschätzte gescannte Bytes" }, "use_id": "ID {{id}} verwenden", "user_id_placeholder": "Benutzer-ID einfügen", diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json index a344ddc38..569e80987 100644 --- a/frontend/src/locales/en.json +++ b/frontend/src/locales/en.json @@ -36,7 +36,8 @@ "view_docs_aria": "View docs (opens in a new tab)", "on_behalf_of": "on behalf of {{name}}", "on_behalf_of_tooltip": "An API-key caller acted for this person (X-AccessFlow-On-Behalf-Of). The named person is treated as a submitter for the self-approval ban.", - "principal_service_account_tooltip": "A non-human identity: it authenticates with an API key only and can never sign in interactively." + "principal_service_account_tooltip": "A non-human identity: it authenticates with an API key only and can never sign in interactively.", + "bytes_unit": "Unit" }, "nav": { "editor": "Query editor", @@ -654,7 +655,8 @@ "affected_rows_label": "Affected rows (exact)", "est_rows_label": "Estimated rows", "scan_type_label": "Scan type", - "cost_label": "Estimated cost" + "cost_label": "Estimated cost", + "est_bytes_label": "Estimated bytes scanned" }, "approval_prediction": { "value_label": "Historical approval likelihood", @@ -864,7 +866,9 @@ "hint": "Effective SQL is the statement as it actually ran, with row-security filters and soft-delete rewrites applied. Bound values are shown as ? and never stored.", "diff_submitted_label": "Submitted SQL", "diff_effective_label": "Effective SQL" - } + }, + "bytes_cap_body": "Estimated scan: {{estimate}} · Cap: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "no estimate" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Environment", "environment_not_set": "Not set (organization default rules)", - "environment_help": "Picks which SQL review ruleset applies to queries on this datasource." + "environment_help": "Picks which SQL review ruleset applies to queries on this datasource.", + "label_max_bytes_scanned": "Max bytes scanned per query", + "max_bytes_scanned_help": "Refuse queries whose pre-flight scan estimate is larger than this. Leave empty for no cap.", + "max_bytes_scanned_placeholder": "No cap", + "label_bytes_cap_missing_estimate": "When no bytes estimate is available", + "bytes_cap_missing_estimate_help": "Applies only while a cap is set. Require human review holds automatic approvals for a person; Reject refuses the query.", + "perm_col_bytes_cap": "Bytes cap", + "perm_bytes_cap_none": "default", + "grant_bytes_cap_label": "Bytes-scanned cap", + "grant_bytes_cap_help": "A lower cap for this grant. The strictest of the datasource cap and every grant applies.", + "grant_bytes_cap_min": "Bytes-scanned cap must be at least 1 byte.", + "grant_bytes_cap_placeholder": "Datasource default" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "e.g. *curl*, *GitHubActions*", "minutes_suffix": "min", "cicd_present_label": "Is CI/CD origin", - "scan_type_placeholder": "e.g. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "e.g. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Bytes scanned" }, "langfuse": { "title": "Langfuse", @@ -3300,7 +3316,8 @@ "time_since_last_approval": "Time since last approval", "cicd_origin": "CI/CD origin", "estimated_rows": "Estimated rows", - "scan_type": "Scan type" + "scan_type": "Scan type", + "estimated_bytes_scanned": "Estimated bytes scanned" }, "query_shape": { "JOIN": "Join", @@ -3740,7 +3757,8 @@ "ELIGIBLE_REVIEWERS": "Eligible reviewers", "ROW_SECURITY": "Row security", "MASKING": "Masking", - "BREAK_GLASS": "Break-glass" + "BREAK_GLASS": "Break-glass", + "BYTES_SCANNED_CAP": "Bytes-scanned cap" }, "api_decision_step": { "CONNECTOR_GATES": "Connector gates", @@ -3795,6 +3813,20 @@ "FIXED_RATE": "Fixed rate", "CRON": "Cron", "ONE_TIME": "One time" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Require human review", + "REJECT": "Reject the query" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Datasource cap", + "GRANT": "Grant override" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Within the bytes-scanned cap", + "EXCEEDED": "Rejected: the estimate exceeds the bytes-scanned cap", + "NO_ESTIMATE_REVIEW": "Held for review: no bytes estimate under the cap", + "NO_ESTIMATE_REJECTED": "Rejected: no bytes estimate under the cap" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Requester user agent", "risk_level": "Risk level", "risk_score": "Risk score", - "scan_type": "Scan type" + "scan_type": "Scan type", + "bytes_cap_suppressed": "Suppressed by the bytes cap", + "bytes_scanned_cap": "Bytes-scanned cap", + "bytes_scanned_cap_outcome": "Cap outcome", + "bytes_scanned_cap_source": "Cap source", + "estimated_bytes_scanned": "Estimated bytes scanned" }, "use_id": "Use ID {{id}}", "user_id_placeholder": "Paste a user ID", diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.json index 2f6b4d179..db432ca89 100644 --- a/frontend/src/locales/es.json +++ b/frontend/src/locales/es.json @@ -36,7 +36,8 @@ "view_docs_aria": "Ver documentación (se abre en una pestaña nueva)", "on_behalf_of": "en nombre de {{name}}", "on_behalf_of_tooltip": "Un llamador con clave de API actuó por esta persona (X-AccessFlow-On-Behalf-Of). La persona nombrada cuenta como solicitante para la prohibición de autoaprobación.", - "principal_service_account_tooltip": "Una identidad no humana: se autentica solo con una clave de API y nunca puede iniciar sesión de forma interactiva." + "principal_service_account_tooltip": "Una identidad no humana: se autentica solo con una clave de API y nunca puede iniciar sesión de forma interactiva.", + "bytes_unit": "Unidad" }, "nav": { "editor": "Editor SQL", @@ -654,7 +655,8 @@ "affected_rows_label": "Filas afectadas (exacto)", "est_rows_label": "Filas estimadas", "scan_type_label": "Tipo de escaneo", - "cost_label": "Coste estimado" + "cost_label": "Coste estimado", + "est_bytes_label": "Bytes escaneados estimados" }, "approval_prediction": { "value_label": "Probabilidad histórica de aprobación", @@ -864,7 +866,9 @@ "hint": "El SQL efectivo es la sentencia tal como se ejecutó realmente, con los filtros de seguridad de filas y las reescrituras de borrado lógico aplicados. Los valores enlazados se muestran como ? y nunca se almacenan.", "diff_submitted_label": "SQL enviado", "diff_effective_label": "SQL efectivo" - } + }, + "bytes_cap_body": "Escaneo estimado: {{estimate}} · Límite: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "sin estimación" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Entorno", "environment_not_set": "Sin definir (reglas predeterminadas de la organización)", - "environment_help": "Determina qué conjunto de reglas de revisión SQL se aplica a las consultas de esta fuente de datos." + "environment_help": "Determina qué conjunto de reglas de revisión SQL se aplica a las consultas de esta fuente de datos.", + "label_max_bytes_scanned": "Máximo de bytes escaneados por consulta", + "max_bytes_scanned_help": "Rechaza las consultas cuya estimación previa de escaneo sea mayor. Déjalo vacío para no poner límite.", + "max_bytes_scanned_placeholder": "Sin límite", + "label_bytes_cap_missing_estimate": "Cuando no hay estimación de bytes", + "bytes_cap_missing_estimate_help": "Solo se aplica mientras haya un límite. Exigir revisión humana retiene las aprobaciones automáticas para una persona; Rechazar deniega la consulta.", + "perm_col_bytes_cap": "Límite de bytes", + "perm_bytes_cap_none": "predeterminado", + "grant_bytes_cap_label": "Límite de bytes escaneados", + "grant_bytes_cap_help": "Un límite menor para este permiso. Se aplica el más estricto entre el límite de la fuente de datos y todos los permisos.", + "grant_bytes_cap_min": "El límite de bytes escaneados debe ser al menos 1 byte.", + "grant_bytes_cap_placeholder": "Predeterminado de la fuente de datos" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "p. ej. *curl*, *GitHubActions*", "minutes_suffix": "min", "cicd_present_label": "Es origen CI/CD", - "scan_type_placeholder": "p. ej. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "p. ej. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Bytes escaneados" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Tiempo desde la última aprobación", "cicd_origin": "Origen CI/CD", "estimated_rows": "Filas estimadas", - "scan_type": "Tipo de escaneo" + "scan_type": "Tipo de escaneo", + "estimated_bytes_scanned": "Bytes escaneados estimados" }, "query_shape": { "JOIN": "Join", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Revisores elegibles", "ROW_SECURITY": "Seguridad por filas", "MASKING": "Enmascaramiento", - "BREAK_GLASS": "Acceso de emergencia" + "BREAK_GLASS": "Acceso de emergencia", + "BYTES_SCANNED_CAP": "Límite de bytes escaneados" }, "api_decision_step": { "CONNECTOR_GATES": "Controles del conector", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Frecuencia fija", "CRON": "Cron", "ONE_TIME": "Única" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Exigir revisión humana", + "REJECT": "Rechazar la consulta" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Límite de la fuente de datos", + "GRANT": "Anulación del permiso" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Dentro del límite de bytes escaneados", + "EXCEEDED": "Rechazada: la estimación supera el límite de bytes escaneados", + "NO_ESTIMATE_REVIEW": "Retenida para revisión: sin estimación de bytes bajo el límite", + "NO_ESTIMATE_REJECTED": "Rechazada: sin estimación de bytes bajo el límite" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Agente de usuario del solicitante", "risk_level": "Nivel de riesgo", "risk_score": "Puntuación de riesgo", - "scan_type": "Tipo de escaneo" + "scan_type": "Tipo de escaneo", + "bytes_cap_suppressed": "Suprimido por el límite de bytes", + "bytes_scanned_cap": "Límite de bytes escaneados", + "bytes_scanned_cap_outcome": "Resultado del límite", + "bytes_scanned_cap_source": "Origen del límite", + "estimated_bytes_scanned": "Bytes escaneados estimados" }, "use_id": "Usar el ID {{id}}", "user_id_placeholder": "Pegue un ID de usuario", diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.json index d68360682..c36d0004c 100644 --- a/frontend/src/locales/fr.json +++ b/frontend/src/locales/fr.json @@ -36,7 +36,8 @@ "view_docs_aria": "Voir la documentation (s'ouvre dans un nouvel onglet)", "on_behalf_of": "pour le compte de {{name}}", "on_behalf_of_tooltip": "Un appelant par clé d'API a agi pour cette personne (X-AccessFlow-On-Behalf-Of). La personne nommée compte comme demandeur pour l'interdiction d'auto-approbation.", - "principal_service_account_tooltip": "Une identité non humaine : elle s'authentifie uniquement par clé d'API et ne peut jamais se connecter de façon interactive." + "principal_service_account_tooltip": "Une identité non humaine : elle s'authentifie uniquement par clé d'API et ne peut jamais se connecter de façon interactive.", + "bytes_unit": "Unité" }, "nav": { "editor": "Éditeur SQL", @@ -654,7 +655,8 @@ "affected_rows_label": "Lignes affectées (exact)", "est_rows_label": "Lignes estimées", "scan_type_label": "Type de scan", - "cost_label": "Coût estimé" + "cost_label": "Coût estimé", + "est_bytes_label": "Octets analysés estimés" }, "approval_prediction": { "value_label": "Probabilité d'approbation historique", @@ -864,7 +866,9 @@ "hint": "Le SQL effectif est l’instruction telle qu’elle a réellement été exécutée, avec les filtres de sécurité des lignes et les réécritures de suppression logique appliqués. Les valeurs liées sont affichées sous forme de ? et ne sont jamais stockées.", "diff_submitted_label": "SQL soumis", "diff_effective_label": "SQL effectif" - } + }, + "bytes_cap_body": "Analyse estimée : {{estimate}} · Limite : {{limit}} ({{source}})", + "bytes_cap_no_estimate": "aucune estimation" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Environnement", "environment_not_set": "Non défini (règles par défaut de l'organisation)", - "environment_help": "Détermine quel jeu de règles de revue SQL s’applique aux requêtes sur cette source de données." + "environment_help": "Détermine quel jeu de règles de revue SQL s’applique aux requêtes sur cette source de données.", + "label_max_bytes_scanned": "Octets analysés max. par requête", + "max_bytes_scanned_help": "Refuse les requêtes dont l’estimation préalable dépasse cette valeur. Laisser vide pour aucune limite.", + "max_bytes_scanned_placeholder": "Aucune limite", + "label_bytes_cap_missing_estimate": "Sans estimation en octets", + "bytes_cap_missing_estimate_help": "S’applique seulement si une limite est définie. « Exiger une revue humaine » soumet les approbations automatiques à une personne ; « Rejeter » refuse la requête.", + "perm_col_bytes_cap": "Limite d’octets", + "perm_bytes_cap_none": "par défaut", + "grant_bytes_cap_label": "Limite d’octets analysés", + "grant_bytes_cap_help": "Une limite plus basse pour cette autorisation. La plus stricte entre la limite de la source de données et toutes les autorisations s’applique.", + "grant_bytes_cap_min": "La limite d’octets analysés doit être d’au moins 1 octet.", + "grant_bytes_cap_placeholder": "Valeur par défaut de la source de données" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "p. ex. *curl*, *GitHubActions*", "minutes_suffix": "min", "cicd_present_label": "Provenance CI/CD", - "scan_type_placeholder": "p. ex. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "p. ex. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Octets analysés" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Temps depuis la dernière approbation", "cicd_origin": "Provenance CI/CD", "estimated_rows": "Lignes estimées", - "scan_type": "Type de scan" + "scan_type": "Type de scan", + "estimated_bytes_scanned": "Octets analysés estimés" }, "query_shape": { "JOIN": "Jointure", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Réviseurs éligibles", "ROW_SECURITY": "Sécurité par ligne", "MASKING": "Masquage", - "BREAK_GLASS": "Accès d'urgence" + "BREAK_GLASS": "Accès d'urgence", + "BYTES_SCANNED_CAP": "Limite d’octets analysés" }, "api_decision_step": { "CONNECTOR_GATES": "Contrôles du connecteur", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Fréquence fixe", "CRON": "Cron", "ONE_TIME": "Unique" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Exiger une revue humaine", + "REJECT": "Rejeter la requête" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Limite de la source de données", + "GRANT": "Dérogation de l’autorisation" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Dans la limite d’octets analysés", + "EXCEEDED": "Rejetée : l’estimation dépasse la limite d’octets analysés", + "NO_ESTIMATE_REVIEW": "Mise en revue : aucune estimation en octets sous la limite", + "NO_ESTIMATE_REJECTED": "Rejetée : aucune estimation en octets sous la limite" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Agent utilisateur du demandeur", "risk_level": "Niveau de risque", "risk_score": "Score de risque", - "scan_type": "Type de parcours" + "scan_type": "Type de parcours", + "bytes_cap_suppressed": "Neutralisé par la limite d’octets", + "bytes_scanned_cap": "Limite d’octets analysés", + "bytes_scanned_cap_outcome": "Résultat de la limite", + "bytes_scanned_cap_source": "Origine de la limite", + "estimated_bytes_scanned": "Octets analysés estimés" }, "use_id": "Utiliser l'ID {{id}}", "user_id_placeholder": "Collez un ID d'utilisateur", diff --git a/frontend/src/locales/hy.json b/frontend/src/locales/hy.json index 582d46bfc..c20dfd2ef 100644 --- a/frontend/src/locales/hy.json +++ b/frontend/src/locales/hy.json @@ -36,7 +36,8 @@ "view_docs_aria": "Դիտել փաստաթղթերը (բացվում է նոր ներդիրում)", "on_behalf_of": "{{name}}-ի անունից", "on_behalf_of_tooltip": "API բանալիով զանգողը գործել է այս անձի անունից (X-AccessFlow-On-Behalf-Of)։ Նշված անձը ինքնահաստատման արգելքի համար համարվում է ներկայացնող։", - "principal_service_account_tooltip": "Ոչ մարդկային ինքնություն. նույնականանում է միայն API բանալիով և երբեք չի կարող ինտերակտիվ մուտք գործել։" + "principal_service_account_tooltip": "Ոչ մարդկային ինքնություն. նույնականանում է միայն API բանալիով և երբեք չի կարող ինտերակտիվ մուտք գործել։", + "bytes_unit": "Միավոր" }, "nav": { "editor": "SQL խմբագիր", @@ -654,7 +655,8 @@ "affected_rows_label": "Ազդված տողեր (ճշգրիտ)", "est_rows_label": "Գնահատված տողեր", "scan_type_label": "Սկանավորման տեսակ", - "cost_label": "Գնահատված արժեք" + "cost_label": "Գնահատված արժեք", + "est_bytes_label": "Սկանավորվող բայթերի գնահատական" }, "approval_prediction": { "value_label": "Հաստատման պատմական հավանականություն", @@ -864,7 +866,9 @@ "hint": "Փաստացի SQL-ը հրահանգն է այնպես, ինչպես այն իրականում կատարվել է՝ տողերի անվտանգության զտիչներով և փափուկ ջնջման վերաշարադրումներով։ Կապված արժեքները ցուցադրվում են որպես ? և երբեք չեն պահպանվում։", "diff_submitted_label": "Ներկայացված SQL", "diff_effective_label": "Փաստացի SQL" - } + }, + "bytes_cap_body": "Գնահատված սկանավորում՝ {{estimate}} · Սահմանաչափ՝ {{limit}} ({{source}})", + "bytes_cap_no_estimate": "գնահատական չկա" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Միջավայր", "environment_not_set": "Սահմանված չէ (կազմակերպության լռելյայն կանոններ)", - "environment_help": "Որոշում է, թե որ SQL ստուգման կանոնակարգն է կիրառվում այս տվյալների աղբյուրի հարցումների վրա։" + "environment_help": "Որոշում է, թե որ SQL ստուգման կանոնակարգն է կիրառվում այս տվյալների աղբյուրի հարցումների վրա։", + "label_max_bytes_scanned": "Սկանավորված բայթերի առավելագույնը մեկ հարցման համար", + "max_bytes_scanned_help": "Մերժել հարցումները, որոնց նախնական սկանավորման գնահատականն ավելի մեծ է։ Թողեք դատարկ՝ առանց սահմանաչափի։", + "max_bytes_scanned_placeholder": "Առանց սահմանաչափի", + "label_bytes_cap_missing_estimate": "Երբ բայթերի գնահատական չկա", + "bytes_cap_missing_estimate_help": "Կիրառվում է միայն սահմանաչափ սահմանված լինելու դեպքում։ «Պահանջել մարդկային ստուգում»-ը ավտոմատ հաստատումները թողնում է մարդուն, «Մերժել»-ը մերժում է հարցումը։", + "perm_col_bytes_cap": "Բայթերի սահմանաչափ", + "perm_bytes_cap_none": "լռելյայն", + "grant_bytes_cap_label": "Սկանավորված բայթերի սահմանաչափ", + "grant_bytes_cap_help": "Ավելի ցածր սահմանաչափ այս թույլտվության համար։ Կիրառվում է տվյալների աղբյուրի և բոլոր թույլտվությունների սահմանաչափերից ամենախիստը։", + "grant_bytes_cap_min": "Սկանավորված բայթերի սահմանաչափը պետք է լինի առնվազն 1 բայթ։", + "grant_bytes_cap_placeholder": "Տվյալների աղբյուրի լռելյայնը" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "օր. *curl*, *GitHubActions*", "minutes_suffix": "ր", "cicd_present_label": "CI/CD ծագում", - "scan_type_placeholder": "օր.՝ Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "օր.՝ Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Սկանավորված բայթեր" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Ժամանակ վերջին հաստատումից", "cicd_origin": "CI/CD ծագում", "estimated_rows": "Գնահատված տողեր", - "scan_type": "Սկանավորման տեսակ" + "scan_type": "Սկանավորման տեսակ", + "estimated_bytes_scanned": "Սկանավորվող բայթերի գնահատական" }, "query_shape": { "JOIN": "Միացում (JOIN)", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Իրավասու վերանայողներ", "ROW_SECURITY": "Տողային անվտանգություն", "MASKING": "Քողարկում", - "BREAK_GLASS": "Արտակարգ մուտք" + "BREAK_GLASS": "Արտակարգ մուտք", + "BYTES_SCANNED_CAP": "Սկանավորված բայթերի սահմանաչափ" }, "api_decision_step": { "CONNECTOR_GATES": "Միակցիչի ստուգումներ", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Ֆիքսված հաճախականություն", "CRON": "Cron", "ONE_TIME": "Միանվագ" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Պահանջել մարդկային ստուգում", + "REJECT": "Մերժել հարցումը" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Տվյալների աղբյուրի սահմանաչափ", + "GRANT": "Թույլտվության վերասահմանում" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Սկանավորված բայթերի սահմանաչափի մեջ է", + "EXCEEDED": "Մերժված է. գնահատականը գերազանցում է սկանավորված բայթերի սահմանաչափը", + "NO_ESTIMATE_REVIEW": "Պահված է ստուգման համար. սահմանաչափի ներքո բայթերի գնահատական չկա", + "NO_ESTIMATE_REJECTED": "Մերժված է. սահմանաչափի ներքո բայթերի գնահատական չկա" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Հայցողի user agent", "risk_level": "Ռիսկի մակարդակ", "risk_score": "Ռիսկի միավոր", - "scan_type": "Սկանավորման տեսակ" + "scan_type": "Սկանավորման տեսակ", + "bytes_cap_suppressed": "Ճնշված է բայթերի սահմանաչափով", + "bytes_scanned_cap": "Սկանավորված բայթերի սահմանաչափ", + "bytes_scanned_cap_outcome": "Սահմանաչափի արդյունք", + "bytes_scanned_cap_source": "Սահմանաչափի աղբյուր", + "estimated_bytes_scanned": "Սկանավորվող բայթերի գնահատական" }, "use_id": "Օգտագործել ID {{id}}", "user_id_placeholder": "Տեղադրեք օգտատիրոջ ID", diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.json index fec35742c..5367102af 100644 --- a/frontend/src/locales/ru.json +++ b/frontend/src/locales/ru.json @@ -36,7 +36,8 @@ "view_docs_aria": "Открыть документацию (в новой вкладке)", "on_behalf_of": "от имени {{name}}", "on_behalf_of_tooltip": "Вызывающий с API-ключом действовал от имени этого человека (X-AccessFlow-On-Behalf-Of). Названный человек считается отправителем для запрета самоодобрения.", - "principal_service_account_tooltip": "Нечеловеческая идентичность: аутентифицируется только API-ключом и никогда не может войти интерактивно." + "principal_service_account_tooltip": "Нечеловеческая идентичность: аутентифицируется только API-ключом и никогда не может войти интерактивно.", + "bytes_unit": "Единица" }, "nav": { "editor": "Редактор SQL", @@ -654,7 +655,8 @@ "affected_rows_label": "Затронутые строки (точно)", "est_rows_label": "Оценка строк", "scan_type_label": "Тип сканирования", - "cost_label": "Оценочная стоимость" + "cost_label": "Оценочная стоимость", + "est_bytes_label": "Оценка сканируемых байтов" }, "approval_prediction": { "value_label": "Историческая вероятность одобрения", @@ -864,7 +866,9 @@ "hint": "Фактический SQL — это запрос в том виде, в котором он был выполнен, с применёнными фильтрами построчной безопасности и перезаписью мягкого удаления. Связанные значения показаны как ? и никогда не сохраняются.", "diff_submitted_label": "Отправленный SQL", "diff_effective_label": "Фактический SQL" - } + }, + "bytes_cap_body": "Оценка сканирования: {{estimate}} · Лимит: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "нет оценки" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Среда", "environment_not_set": "Не задано (правила организации по умолчанию)", - "environment_help": "Определяет, какой набор правил проверки SQL применяется к запросам к этому источнику данных." + "environment_help": "Определяет, какой набор правил проверки SQL применяется к запросам к этому источнику данных.", + "label_max_bytes_scanned": "Макс. сканируемых байтов на запрос", + "max_bytes_scanned_help": "Отклонять запросы, чья предварительная оценка сканирования больше. Оставьте пустым, чтобы не ограничивать.", + "max_bytes_scanned_placeholder": "Без лимита", + "label_bytes_cap_missing_estimate": "Если оценки в байтах нет", + "bytes_cap_missing_estimate_help": "Действует только при заданном лимите. «Требовать проверку человеком» оставляет автоматические одобрения человеку; «Отклонить» отклоняет запрос.", + "perm_col_bytes_cap": "Лимит байтов", + "perm_bytes_cap_none": "по умолчанию", + "grant_bytes_cap_label": "Лимит сканируемых байтов", + "grant_bytes_cap_help": "Более низкий лимит для этой выдачи доступа. Применяется самый строгий из лимита источника данных и всех выдач.", + "grant_bytes_cap_min": "Лимит сканируемых байтов должен быть не менее 1 байта.", + "grant_bytes_cap_placeholder": "По умолчанию источника данных" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "напр. *curl*, *GitHubActions*", "minutes_suffix": "мин", "cicd_present_label": "Источник CI/CD", - "scan_type_placeholder": "напр. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "напр. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Сканируемые байты" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Время с последнего одобрения", "cicd_origin": "Источник CI/CD", "estimated_rows": "Оценка строк", - "scan_type": "Тип сканирования" + "scan_type": "Тип сканирования", + "estimated_bytes_scanned": "Оценка сканируемых байтов" }, "query_shape": { "JOIN": "Соединение (JOIN)", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Допустимые проверяющие", "ROW_SECURITY": "Построчная безопасность", "MASKING": "Маскирование", - "BREAK_GLASS": "Экстренный доступ" + "BREAK_GLASS": "Экстренный доступ", + "BYTES_SCANNED_CAP": "Лимит сканируемых байтов" }, "api_decision_step": { "CONNECTOR_GATES": "Проверки коннектора", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Фиксированная частота", "CRON": "Cron", "ONE_TIME": "Однократно" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Требовать проверку человеком", + "REJECT": "Отклонить запрос" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Лимит источника данных", + "GRANT": "Переопределение выдачи доступа" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "В пределах лимита сканируемых байтов", + "EXCEEDED": "Отклонено: оценка превышает лимит сканируемых байтов", + "NO_ESTIMATE_REVIEW": "Отправлено на проверку: нет оценки в байтах при лимите", + "NO_ESTIMATE_REJECTED": "Отклонено: нет оценки в байтах при лимите" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "User agent автора", "risk_level": "Уровень риска", "risk_score": "Оценка риска", - "scan_type": "Тип сканирования" + "scan_type": "Тип сканирования", + "bytes_cap_suppressed": "Подавлено лимитом байтов", + "bytes_scanned_cap": "Лимит сканируемых байтов", + "bytes_scanned_cap_outcome": "Результат лимита", + "bytes_scanned_cap_source": "Источник лимита", + "estimated_bytes_scanned": "Оценка сканируемых байтов" }, "use_id": "Использовать ID {{id}}", "user_id_placeholder": "Вставьте ID пользователя", diff --git a/frontend/src/locales/zh-CN.json b/frontend/src/locales/zh-CN.json index 11dfabd35..ea8a8cb5f 100644 --- a/frontend/src/locales/zh-CN.json +++ b/frontend/src/locales/zh-CN.json @@ -36,7 +36,8 @@ "view_docs_aria": "查看文档（在新标签页中打开）", "on_behalf_of": "代表 {{name}}", "on_behalf_of_tooltip": "某个 API 密钥调用方代表此人执行了操作（X-AccessFlow-On-Behalf-Of）。在禁止自行审批的规则中，被指名的人视为提交者。", - "principal_service_account_tooltip": "非人类身份：仅通过 API 密钥认证，永远无法交互式登录。" + "principal_service_account_tooltip": "非人类身份：仅通过 API 密钥认证，永远无法交互式登录。", + "bytes_unit": "单位" }, "nav": { "editor": "SQL 编辑器", @@ -654,7 +655,8 @@ "affected_rows_label": "受影响行数（精确）", "est_rows_label": "预估行数", "scan_type_label": "扫描类型", - "cost_label": "预估成本" + "cost_label": "预估成本", + "est_bytes_label": "预计扫描字节数" }, "approval_prediction": { "value_label": "历史批准可能性", @@ -864,7 +866,9 @@ "hint": "实际执行的 SQL 是语句真正运行时的形式，已应用行级安全过滤和软删除改写。绑定值显示为 ?，且从不存储。", "diff_submitted_label": "提交的 SQL", "diff_effective_label": "实际执行的 SQL" - } + }, + "bytes_cap_body": "预计扫描：{{estimate}} · 上限：{{limit}}（{{source}}）", + "bytes_cap_no_estimate": "无估算" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "环境", "environment_not_set": "未设置（使用组织默认规则）", - "environment_help": "决定对此数据源的查询应用哪个 SQL 审查规则集。" + "environment_help": "决定对此数据源的查询应用哪个 SQL 审查规则集。", + "label_max_bytes_scanned": "每次查询最大扫描字节数", + "max_bytes_scanned_help": "拒绝预检扫描估算大于此值的查询。留空表示不设上限。", + "max_bytes_scanned_placeholder": "无上限", + "label_bytes_cap_missing_estimate": "没有字节估算时", + "bytes_cap_missing_estimate_help": "仅在设置了上限时生效。“要求人工审查”会把自动批准交给人工处理；“拒绝”会拒绝查询。", + "perm_col_bytes_cap": "字节上限", + "perm_bytes_cap_none": "默认", + "grant_bytes_cap_label": "扫描字节上限", + "grant_bytes_cap_help": "为此授权设置更低的上限。取数据源上限与所有授权中最严格的一个。", + "grant_bytes_cap_min": "扫描字节上限至少为 1 字节。", + "grant_bytes_cap_placeholder": "数据源默认值" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "例如 *curl*、*GitHubActions*", "minutes_suffix": "分钟", "cicd_present_label": "来自 CI/CD", - "scan_type_placeholder": "例如 Seq Scan、COLLSCAN、Index*" + "scan_type_placeholder": "例如 Seq Scan、COLLSCAN、Index*", + "bytes_value_label": "扫描字节数" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "距上次批准时长", "cicd_origin": "CI/CD 来源", "estimated_rows": "预估行数", - "scan_type": "扫描类型" + "scan_type": "扫描类型", + "estimated_bytes_scanned": "预计扫描字节数" }, "query_shape": { "JOIN": "连接（JOIN）", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "合格审查人", "ROW_SECURITY": "行级安全", "MASKING": "脱敏", - "BREAK_GLASS": "紧急访问" + "BREAK_GLASS": "紧急访问", + "BYTES_SCANNED_CAP": "扫描字节上限" }, "api_decision_step": { "CONNECTOR_GATES": "连接器检查", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "固定频率", "CRON": "Cron", "ONE_TIME": "一次性" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "要求人工审查", + "REJECT": "拒绝查询" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "数据源上限", + "GRANT": "授权覆盖" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "在扫描字节上限之内", + "EXCEEDED": "已拒绝：估算超过扫描字节上限", + "NO_ESTIMATE_REVIEW": "已转人工审查：上限下没有字节估算", + "NO_ESTIMATE_REJECTED": "已拒绝：上限下没有字节估算" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "请求者 User-Agent", "risk_level": "风险等级", "risk_score": "风险分数", - "scan_type": "扫描类型" + "scan_type": "扫描类型", + "bytes_cap_suppressed": "被字节上限抑制", + "bytes_scanned_cap": "扫描字节上限", + "bytes_scanned_cap_outcome": "上限结果", + "bytes_scanned_cap_source": "上限来源", + "estimated_bytes_scanned": "预计扫描字节数" }, "use_id": "使用 ID {{id}}", "user_id_placeholder": "粘贴用户 ID", diff --git a/frontend/src/pages/admin/RoutingPoliciesPage.tsx b/frontend/src/pages/admin/RoutingPoliciesPage.tsx index 72607cd0f..6bdbc4b9c 100644 --- a/frontend/src/pages/admin/RoutingPoliciesPage.tsx +++ b/frontend/src/pages/admin/RoutingPoliciesPage.tsx @@ -33,6 +33,7 @@ import { } from '@/components/policies/policyImpact'; import { simulateRoutingPolicy } from '@/api/policySimulation'; import { routingSimulationSummary } from './routingSimulationSummary'; +import { BytesInput } from '@/components/common/BytesInput'; import { PageHeader } from '@/components/common/PageHeader'; import { EmptyState } from '@/components/common/EmptyState'; import { Pill } from '@/components/common/Pill'; @@ -942,6 +943,21 @@ function ConditionValueEditor({ name, operand, groups, roleOptions }: ConditionV
); + case 'estimated_bytes_scanned': + return ( +
+ + ({ + value: v, + label: bytesCapMissingEstimateLabel(t, v), + }))} + /> + + + )} @@ -499,9 +508,21 @@ function ConfigTab({ ds, onDelete, deletePending }: ConfigTabProps) { }); const onFinish = (values: SettingsFormValues) => { - const { read_replicas: replicaRows, environment, ...rest } = values; + const { + read_replicas: replicaRows, + environment, + max_bytes_scanned_per_query: bytesCap, + bytes_cap_missing_estimate: bytesCapMissingEstimate, + ...rest + } = values; // "Not set" must clear explicitly: a null environment means "unchanged" to the API (#861). const body: UpdateDatasourceInput = { ...rest, ...toEnvironmentUpdate(environment) }; + // The cap fields exist only on bytes-reporting engines; anywhere else they are never sent (#941). + if (bytesCapSupported) { + Object.assign(body, toBytesCapUpdate(bytesCap, ds.max_bytes_scanned_per_query), { + bytes_cap_missing_estimate: bytesCapMissingEstimate, + }); + } if (!body.password || body.password.trim().length === 0) { delete body.password; } @@ -803,6 +824,37 @@ function ConfigTab({ ds, onDelete, deletePending }: ConfigTabProps) { }))} /> + {bytesCapSupported && ( + <> + {/* Optional; mirrors the backend @Min(1) (#941). Empty = no cap. */} + + + + +