From d1715dc261effcf90eea251ca80940fa38959ddb Mon Sep 17 00:00:00 2001 From: Tigran Babloyan Date: Fri, 25 Sep 2026 13:20:25 +0400 Subject: [PATCH 1/4] feat(AF-941): bytes-scanned cost caps and estimated_bytes_scanned routing Persist the warehouse pre-flight bytes estimate (V192) and let admins cap it per datasource and per grant (most restrictive wins), refused with 422 on engines without a bytes estimate. The cap rejects before routing, holds auto-approvals for review when no estimate exists (REQUIRE_REVIEW) and is re-checked before execution, groups and break-glass included. Adds the fail-closed estimated_bytes_scanned routing condition, a BYTES_SCANNED_CAP trace step and QUERY_BYTES_SCANNED_CAP_ENFORCED audit rows. Estimate prep runs in its own transaction so a lost insert race never strands a query. --- CLAUDE.md | 12 +- .../internal/AccessGrantMaterializer.java | 4 +- .../ai/internal/DefaultAiAnalyzerService.java | 9 + .../DefaultAttestationLifecycleService.java | 5 + .../accessflow/audit/api/AuditAction.java | 8 + .../accessflow/core/api/AppliedBytesCap.java | 36 ++ .../accessflow/core/api/ByteSizeFormat.java | 32 ++ .../api/BytesCapMissingEstimateAction.java | 12 + .../api/BytesScannedCapExceededException.java | 34 ++ .../BytesScannedCapNotSupportedException.java | 19 + .../core/api/BytesScannedCapOutcome.java | 18 + .../api/BytesScannedCapResolutionService.java | 13 + .../core/api/BytesScannedCapSource.java | 9 + .../core/api/BytesScannedCapSupport.java | 26 ++ .../core/api/CreateDatasourceCommand.java | 29 +- ...reateDatasourceGroupPermissionCommand.java | 1 + .../core/api/CreatePermissionCommand.java | 1 + .../core/api/DatasourceAdminException.java | 1 + .../api/DatasourceGroupPermissionView.java | 1 + .../api/DatasourcePermissionContribution.java | 17 + .../core/api/DatasourcePermissionView.java | 1 + .../api/DatasourceUserPermissionView.java | 14 + .../accessflow/core/api/DatasourceView.java | 28 +- .../core/api/PersistQueryEstimateCommand.java | 4 +- .../accessflow/core/api/QueryDetailView.java | 52 ++- .../core/api/QueryEstimateSnapshot.java | 4 +- .../core/api/QueryRequestStateService.java | 8 + .../core/api/UpdateDatasourceCommand.java | 33 +- .../core/events/QueryAutoRejectedEvent.java | 7 +- .../internal/DatasourceAdminServiceImpl.java | 39 +- ...faultBytesScannedCapResolutionService.java | 49 +++ ...DatasourceUserPermissionLookupService.java | 20 +- .../internal/DefaultQueryEstimateService.java | 2 + .../DefaultQueryRequestLookupService.java | 11 +- .../DefaultQueryRequestStateService.java | 14 + .../persistence/entity/DatasourceEntity.java | 12 + .../DatasourceGroupPermissionEntity.java | 4 + .../DatasourceUserPermissionEntity.java | 4 + .../entity/QueryEstimateEntity.java | 3 + .../entity/QueryRequestEntity.java | 17 + .../proxy/api/QueryCostEstimateService.java | 10 + .../DefaultQueryCostEstimateService.java | 26 +- .../internal/GroupExecutionService.java | 73 +++- .../repo/GroupReviewDecisionRepository.java | 3 + .../internal/web/DatasourceController.java | 21 +- .../internal/web/GlobalExceptionHandler.java | 11 + .../web/model/CreateDatasourceRequest.java | 6 +- .../model/CreateGroupPermissionRequest.java | 2 + .../web/model/CreatePermissionRequest.java | 2 + .../web/model/DatasourceResponse.java | 9 +- .../web/model/GroupPermissionResponse.java | 2 + .../web/model/PermissionResponse.java | 2 + .../web/model/UpdateDatasourceRequest.java | 7 +- .../workflow/api/ConditionContext.java | 24 +- .../workflow/api/ConditionNode.java | 19 + .../workflow/api/QueryDecisionStepKind.java | 11 +- .../workflow/internal/BytesCapCheck.java | 42 +++ .../DefaultAccessSimulationService.java | 11 +- .../DefaultQueryLifecycleService.java | 68 ++++ .../workflow/internal/QueryDecision.java | 17 +- .../internal/QueryDecisionEvaluator.java | 147 ++++++-- .../workflow/internal/QueryDecisionKind.java | 8 +- .../internal/QueryReviewStateMachine.java | 148 +++++++- .../routing/ConditionContextFactory.java | 14 +- .../internal/routing/ConditionNodeMixin.java | 1 + .../routing/RoutingConditionEvaluator.java | 2 + .../web/AccessSimulationResponse.java | 6 +- .../internal/web/QueryDetailResponse.java | 28 +- .../V192__add_bytes_scanned_caps.sql | 25 ++ .../main/resources/i18n/messages.properties | 21 ++ .../resources/i18n/messages_de.properties | 21 ++ .../resources/i18n/messages_es.properties | 21 ++ .../resources/i18n/messages_fr.properties | 21 ++ .../resources/i18n/messages_hy.properties | 21 ++ .../resources/i18n/messages_ru.properties | 21 ++ .../resources/i18n/messages_zh_CN.properties | 21 ++ .../internal/AccessGrantMaterializerTest.java | 21 +- ...faultGrantUsageAggregationServiceTest.java | 2 +- .../DefaultAiAnalyzerServiceTest.java | 22 ++ .../AttestationLifecycleIntegrationTest.java | 2 +- ...efaultAttestationLifecycleServiceTest.java | 8 +- .../core/api/BytesScannedCapRecordsTest.java | 90 +++++ .../DatasourceAdminServiceImplTest.java | 149 +++++++- ...tBytesScannedCapResolutionServiceTest.java | 126 +++++++ ...sourceUserPermissionLookupServiceTest.java | 45 +++ .../DefaultQueryEstimateServiceTest.java | 2 +- .../DefaultQueryRequestLookupServiceTest.java | 46 +++ .../DefaultQueryRequestStateServiceTest.java | 19 + .../DefaultQueryCostEstimateServiceTest.java | 52 ++- .../internal/GroupExecutionServiceTest.java | 123 ++++++ .../DatasourceControllerIntegrationTest.java | 95 +++++ .../web/GlobalExceptionHandlerTest.java | 12 + .../workflow/api/ConditionNodeTest.java | 20 + .../workflow/internal/BytesCapCheckTest.java | 48 +++ ...sScannedCapEnforcementIntegrationTest.java | 355 ++++++++++++++++++ .../DefaultAccessSimulationServiceTest.java | 41 +- .../DefaultQueryLifecycleServiceTest.java | 108 +++++- .../internal/QueryDecisionEvaluatorTest.java | 162 +++++++- .../internal/QueryReviewStateMachineTest.java | 176 ++++++++- .../routing/ConditionContextFactoryTest.java | 19 +- .../routing/RoutingConditionCodecTest.java | 12 + .../RoutingConditionEvaluatorTest.java | 22 ++ ...ssSimulationControllerIntegrationTest.java | 15 +- .../internal/web/QueryDetailResponseTest.java | 30 ++ 104 files changed, 3195 insertions(+), 132 deletions(-) create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java create mode 100644 backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java create mode 100644 backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql create mode 100644 backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java create mode 100644 backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java create mode 100644 backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java create mode 100644 backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java diff --git a/CLAUDE.md b/CLAUDE.md index ab00aa057..9c4f1983c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -341,6 +341,14 @@ nullable or has a DEFAULT. `ALTER TYPE … ADD VALUE` needs a `.sql.conf` sideca once as SQL_REVIEW_BLOCKED when it changed the outcome) PENDING_AI → REJECTED (routing-policy AUTO_REJECT — AF-379; no review_decisions row, audited via QueryAutoRejectedEvent) + PENDING_AI → REJECTED (bytes-scanned cap — #941; the warehouse's pre-flight bytes estimate + exceeds the datasource/grant cap, or there is none and the datasource's + bytes_cap_missing_estimate=REJECT. Decided before routing and the + AI-failed path, no routing_decision row, QueryAutoRejectedEvent with a + null policy id; audited as QUERY_BYTES_SCANNED_CAP_ENFORCED) + PENDING_AI → PENDING_REVIEW (bytes-scanned cap, no estimate, bytes_cap_missing_estimate= + REQUIRE_REVIEW — #941; suppresses the same auto-approve paths as a SQL + review BLOCK, never softens AUTO_REJECT) PENDING_REVIEW → APPROVED or REJECTED (external ticket resolution — AF-453; a channel with bidirectional_sync=true maps a ServiceNow/Jira ticket resolution onto a decision via workflow.api.ExternalDecisionService. System-attributed: @@ -363,7 +371,9 @@ nullable or has a DEFAULT. `ALTER TYPE … ADD VALUE` needs a `.sql.conf` sideca recurrence_next_run_at, records recurrence_halted_reason, and audits RECURRING_SERIES_HALTED) APPROVED → EXECUTED (break-glass run — audit action QUERY_BREAK_GLASS_EXECUTED — AF-385) - APPROVED → FAILED (execution error) + APPROVED → FAILED (execution error; also the bytes-scanned cap re-checked just before + execution refusing the run — #941, scheduled / recurring / + break-glass included) ``` Illegal transitions must throw a domain exception, not silently succeed. **Break-glass / diff --git a/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java b/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java index 1a400e155..c191cc949 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java +++ b/backend/src/main/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializer.java @@ -54,7 +54,8 @@ void materialize(UUID accessRequestId, UUID approvedByUserId) { return; } // A replaced row's denials carry over (#939): a JIT approval widens capabilities and expiry, - // never lifts a denial an admin set — JIT requests cannot even ask for deny-lists. + // never lifts a denial an admin set — JIT requests cannot even ask for deny-lists. The + // bytes-scanned cap carries over for the same reason (#941): a JIT approval never widens cost. var replaced = replaceExistingTimeBoxedPermission(entity); var command = new CreatePermissionCommand( entity.getRequesterId(), @@ -63,6 +64,7 @@ void materialize(UUID accessRequestId, UUID approvedByUserId) { entity.isCanDdl(), false, null, + replaced.map(DatasourceUserPermissionView::bytesScannedLimitOverride).orElse(null), toList(entity.getAllowedSchemas()), toList(entity.getAllowedTables()), null, diff --git a/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java b/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java index 7f91db21b..94473aaa9 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerService.java @@ -8,6 +8,7 @@ import com.bablsoft.accessflow.ai.api.AiBudgetExceededException; import com.bablsoft.accessflow.ai.api.AiRateLimitExceededException; import com.bablsoft.accessflow.ai.internal.persistence.repo.AiConfigRepository; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; import com.bablsoft.accessflow.core.api.AiAnalysisPersistenceService; import com.bablsoft.accessflow.core.api.AiProviderType; import com.bablsoft.accessflow.core.api.DataClassification; @@ -296,6 +297,14 @@ private String buildCostEstimateContext(UUID queryRequestId) { } sb.append("Plan root operation: ").append(estimate.scanType()).append('.'); } + if (estimate.estimatedBytesScanned() != null) { + if (!sb.isEmpty()) { + sb.append(' '); + } + sb.append("The warehouse estimates this query will scan ") + .append(ByteSizeFormat.format(estimate.estimatedBytesScanned())) + .append('.'); + } return sb.isEmpty() ? null : sb.toString(); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java b/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java index 16684e151..05c0cce50 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleService.java @@ -211,6 +211,11 @@ private String toSnapshotJson(DatasourcePermissionView view) { } else { node.putNull("row_limit_override"); } + if (view.bytesScannedLimitOverride() != null) { + node.put("bytes_scanned_limit_override", view.bytesScannedLimitOverride()); + } else { + node.putNull("bytes_scanned_limit_override"); + } putStringArray(node, "allowed_schemas", view.allowedSchemas()); putStringArray(node, "allowed_tables", view.allowedTables()); putStringArray(node, "restricted_columns", view.restrictedColumns()); diff --git a/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java b/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java index 1d19595a8..415bec54b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java +++ b/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java @@ -132,6 +132,14 @@ public enum AuditAction { * when the guard changed the outcome; never for {@code WARN}, never on a rejection. */ SQL_REVIEW_BLOCKED, + /** + * The bytes-scanned cap (#941) changed a query's outcome: refused it when it left + * {@code PENDING_AI} ({@code stage=decision}), turned an automatic approval into human review, + * or failed it just before execution ({@code stage=execution}). System-attributed — null actor, + * {@code trigger=bytes_scanned_cap}; metadata carries {@code limit}, {@code source}, + * {@code estimated_bytes} and {@code outcome}. + */ + QUERY_BYTES_SCANNED_CAP_ENFORCED, ROW_SECURITY_POLICY_CREATED, ROW_SECURITY_POLICY_UPDATED, ROW_SECURITY_POLICY_DELETED, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java new file mode 100644 index 000000000..1079df1bb --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/AppliedBytesCap.java @@ -0,0 +1,36 @@ +package com.bablsoft.accessflow.core.api; + +import java.util.Objects; + +/** + * The bytes-scanned cap (#941) binding one user on one datasource: the most restrictive of the + * datasource's {@code max_bytes_scanned_per_query} and the user's merged grant override, together + * with the datasource's missing-estimate policy. + */ +public record AppliedBytesCap(long limit, BytesScannedCapSource source, + BytesCapMissingEstimateAction missingEstimate) { + + public AppliedBytesCap { + if (limit <= 0) { + throw new IllegalArgumentException("limit must be positive"); + } + Objects.requireNonNull(source, "source"); + missingEstimate = missingEstimate == null + ? BytesCapMissingEstimateAction.REQUIRE_REVIEW : missingEstimate; + } + + /** + * Compares an estimate against the cap. {@code null} means no estimate exists; the + * datasource's missing-estimate policy then decides. + */ + public BytesScannedCapOutcome check(Long estimatedBytesScanned) { + if (estimatedBytesScanned == null) { + return missingEstimate == BytesCapMissingEstimateAction.REJECT + ? BytesScannedCapOutcome.NO_ESTIMATE_REJECTED + : BytesScannedCapOutcome.NO_ESTIMATE_REVIEW; + } + return estimatedBytesScanned > limit + ? BytesScannedCapOutcome.EXCEEDED + : BytesScannedCapOutcome.WITHIN; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java new file mode 100644 index 000000000..74f452963 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/ByteSizeFormat.java @@ -0,0 +1,32 @@ +package com.bablsoft.accessflow.core.api; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** + * Renders a raw byte count for a user-facing message, in decimal units (the unit warehouses bill + * in), keeping the exact figure alongside so a value just over a limit never reads as equal to it: + * {@code 1.5 TB (1500000000000 B)} — the unit symbols need no translation inside a localized message. Mirrors the frontend's {@code formatBytes}. + */ +public final class ByteSizeFormat { + + private static final String[] UNITS = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; + + private ByteSizeFormat() { + } + + public static String format(long bytes) { + if (bytes < 1000) { + return bytes + " B"; + } + var scaled = BigDecimal.valueOf(bytes); + var thousand = BigDecimal.valueOf(1000); + int unit = 0; + while (scaled.compareTo(thousand) >= 0 && unit < UNITS.length - 1) { + scaled = scaled.divide(thousand); + unit++; + } + var rounded = scaled.setScale(2, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + return rounded + " " + UNITS[unit] + " (" + bytes + " B)"; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java new file mode 100644 index 000000000..6269b6e99 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesCapMissingEstimateAction.java @@ -0,0 +1,12 @@ +package com.bablsoft.accessflow.core.api; + +/** + * What a bytes-scanned cap (#941) does when the query has no bytes estimate to compare against — + * the estimate failed, the statement shape has no plan (DDL), or the engine returned no bytes + * figure. Never "allow silently": {@link #REQUIRE_REVIEW} suppresses every automatic approval so a + * person decides, {@link #REJECT} refuses the query outright. + */ +public enum BytesCapMissingEstimateAction { + REQUIRE_REVIEW, + REJECT +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java new file mode 100644 index 000000000..2e4f18d3c --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapExceededException.java @@ -0,0 +1,34 @@ +package com.bablsoft.accessflow.core.api; + +/** + * The bytes-scanned cap (#941) refused a statement just before execution — its pre-flight + * estimate exceeds the cap, or it has none and the datasource rejects on a missing estimate. The + * message is the caller's localized explanation naming the estimate and the limit; execution paths + * record it as the failure reason. + */ +public class BytesScannedCapExceededException extends RuntimeException { + + private final transient AppliedBytesCap cap; + private final Long estimatedBytes; + private final BytesScannedCapOutcome outcome; + + public BytesScannedCapExceededException(String message, AppliedBytesCap cap, + Long estimatedBytes, BytesScannedCapOutcome outcome) { + super(message); + this.cap = cap; + this.estimatedBytes = estimatedBytes; + this.outcome = outcome; + } + + public AppliedBytesCap cap() { + return cap; + } + + public Long estimatedBytes() { + return estimatedBytes; + } + + public BytesScannedCapOutcome outcome() { + return outcome; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java new file mode 100644 index 000000000..4bbfe770a --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapNotSupportedException.java @@ -0,0 +1,19 @@ +package com.bablsoft.accessflow.core.api; + +/** + * A bytes-scanned cap (#941) — on a datasource or on a grant — was configured for an engine whose + * dry-run does not report a bytes estimate. See {@link BytesScannedCapSupport}. + */ +public final class BytesScannedCapNotSupportedException extends DatasourceAdminException { + + private final DbType dbType; + + public BytesScannedCapNotSupportedException(DbType dbType) { + super("bytes-scanned caps are not supported for db_type " + dbType); + this.dbType = dbType; + } + + public DbType dbType() { + return dbType; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java new file mode 100644 index 000000000..1845463e1 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapOutcome.java @@ -0,0 +1,18 @@ +package com.bablsoft.accessflow.core.api; + +/** How a query's bytes estimate compared against the cap that applied to it (#941). */ +public enum BytesScannedCapOutcome { + /** An estimate existed and was at or below the cap. */ + WITHIN, + /** The estimate was above the cap: the query is refused. */ + EXCEEDED, + /** No estimate, and the datasource says {@link BytesCapMissingEstimateAction#REQUIRE_REVIEW}. */ + NO_ESTIMATE_REVIEW, + /** No estimate, and the datasource says {@link BytesCapMissingEstimateAction#REJECT}. */ + NO_ESTIMATE_REJECTED; + + /** Whether this outcome refuses the query. */ + public boolean rejects() { + return this == EXCEEDED || this == NO_ESTIMATE_REJECTED; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java new file mode 100644 index 000000000..172037384 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapResolutionService.java @@ -0,0 +1,13 @@ +package com.bablsoft.accessflow.core.api; + +import java.util.Optional; +import java.util.UUID; + +/** + * Resolves the bytes-scanned cap (#941) that binds a user on a datasource. Empty when neither the + * datasource nor any of the user's grants sets one. + */ +public interface BytesScannedCapResolutionService { + + Optional resolve(UUID datasourceId, UUID userId); +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java new file mode 100644 index 000000000..09c787239 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSource.java @@ -0,0 +1,9 @@ +package com.bablsoft.accessflow.core.api; + +/** Which configuration supplied the binding bytes-scanned cap (#941). */ +public enum BytesScannedCapSource { + /** {@code datasources.max_bytes_scanned_per_query}. */ + DATASOURCE, + /** A {@code bytes_scanned_limit_override} on the submitter's direct or group grant. */ + GRANT +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java new file mode 100644 index 000000000..14c8a9477 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/BytesScannedCapSupport.java @@ -0,0 +1,26 @@ +package com.bablsoft.accessflow.core.api; + +import java.util.Set; + +/** + * The engines whose pre-flight dry-run reports a bytes-scanned estimate (AF-634), and therefore + * the only ones a bytes-scanned cap (#941) may be configured on. On any other engine every query + * would have no estimate, so a cap there would either reject everything or be a no-op — both + * surprising — and configuring one is refused instead. + */ +public final class BytesScannedCapSupport { + + private static final Set SUPPORTED = Set.of(DbType.BIGQUERY, DbType.SNOWFLAKE, + DbType.DATABRICKS); + + private BytesScannedCapSupport() { + } + + public static boolean supports(DbType dbType) { + return dbType != null && SUPPORTED.contains(dbType); + } + + public static Set supportedTypes() { + return SUPPORTED; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java index c2315faa3..3ef551745 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceCommand.java @@ -30,8 +30,32 @@ public record CreateDatasourceCommand( Boolean resultCacheEnabled, Integer resultCacheTtlSeconds, String privateKeyPassphrase, - DatasourceEnvironment environment + DatasourceEnvironment environment, + Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { + /** + * Backward-compatible constructor for the pre-#941 canonical shape (no bytes-scanned cap); + * delegates with {@code null} — no cap, default missing-estimate policy. + */ + public CreateDatasourceCommand( + UUID organizationId, String name, DbType dbType, String host, Integer port, + String databaseName, String username, String password, SslMode sslMode, + Integer connectionPoolSize, Integer maxRowsPerQuery, Boolean requireReviewReads, + Boolean requireReviewWrites, UUID reviewPlanId, Boolean aiAnalysisEnabled, + UUID aiConfigId, Boolean textToSqlEnabled, UUID customDriverId, String connectorId, + String jdbcUrlOverride, List readReplicas, + String localDatacenter, String apiKey, Boolean resultCacheEnabled, + Integer resultCacheTtlSeconds, String privateKeyPassphrase, + DatasourceEnvironment environment) { + this(organizationId, name, dbType, host, port, databaseName, username, password, sslMode, + connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, + reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, + connectorId, jdbcUrlOverride, readReplicas, localDatacenter, apiKey, + resultCacheEnabled, resultCacheTtlSeconds, privateKeyPassphrase, environment, + null, null); + } + /** * Backward-compatible constructor for the pre-#861 canonical shape (no {@code environment}); * delegates with {@code null} — an unset environment is a legitimate state. @@ -49,7 +73,8 @@ public CreateDatasourceCommand( connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, connectorId, jdbcUrlOverride, readReplicas, localDatacenter, apiKey, - resultCacheEnabled, resultCacheTtlSeconds, privateKeyPassphrase, null); + resultCacheEnabled, resultCacheTtlSeconds, privateKeyPassphrase, + (DatasourceEnvironment) null); } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java index dafda393d..d6ec918ec 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreateDatasourceGroupPermissionCommand.java @@ -11,6 +11,7 @@ public record CreateDatasourceGroupPermissionCommand( Boolean canDdl, Boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java index 242e6c9d7..f405ae08c 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/CreatePermissionCommand.java @@ -15,6 +15,7 @@ public record CreatePermissionCommand( Boolean canDdl, Boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java index 5478b9a0b..85851d0eb 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceAdminException.java @@ -10,6 +10,7 @@ public sealed class DatasourceAdminException extends RuntimeException IllegalDatasourcePermissionException, DeniedColumnsNotSupportedException, DeniedShapesNotSupportedException, + BytesScannedCapNotSupportedException, MissingAiConfigForDatasourceException, TableNotFoundException { diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java index 45c8f71e6..14c6dd307 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceGroupPermissionView.java @@ -15,6 +15,7 @@ public record DatasourceGroupPermissionView( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java index 93fbafec0..5005ed900 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionContribution.java @@ -17,6 +17,8 @@ * row id * @param groupId the granting group, or {@code null} for a direct grant * @param rowLimitOverride this grant's row cap, or {@code null} when it sets none (#933) + * @param bytesScannedLimitOverride this grant's bytes-scanned cap, or {@code null} when it sets + * none (#941) * @param expiresAt {@code null} means this contribution never expires * @param accessGrantRequestId the JIT {@code access_grant_request} a direct row materialises * (#969); {@code null} on an admin-created row and always on a @@ -41,9 +43,24 @@ public record DatasourcePermissionContribution( List deniedTables, List deniedShapes, Integer rowLimitOverride, + Long bytesScannedLimitOverride, Instant expiresAt, UUID accessGrantRequestId) { + /** Backward-compatible constructor without the #941 bytes-scanned cap override. */ + public DatasourcePermissionContribution( + DatasourcePermissionSourceKind sourceKind, UUID sourceId, UUID userId, + UUID datasourceId, UUID groupId, String groupName, boolean canRead, boolean canWrite, + boolean canDdl, boolean canBreakGlass, List allowedSchemas, + List allowedTables, List restrictedColumns, List deniedColumns, + List deniedSchemas, List deniedTables, List deniedShapes, + Integer rowLimitOverride, Instant expiresAt, UUID accessGrantRequestId) { + this(sourceKind, sourceId, userId, datasourceId, groupId, groupName, canRead, canWrite, + canDdl, canBreakGlass, allowedSchemas, allowedTables, restrictedColumns, + deniedColumns, deniedSchemas, deniedTables, deniedShapes, rowLimitOverride, null, + expiresAt, accessGrantRequestId); + } + public DatasourcePermissionContribution { allowedSchemas = allowedSchemas == null ? List.of() : List.copyOf(allowedSchemas); allowedTables = allowedTables == null ? List.of() : List.copyOf(allowedTables); diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java index 6b8a17a5b..cdf278f54 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourcePermissionView.java @@ -15,6 +15,7 @@ public record DatasourcePermissionView( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java index be019dec1..a9a6e6d68 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceUserPermissionView.java @@ -20,5 +20,19 @@ public record DatasourceUserPermissionView( List deniedTables, List deniedShapes, Integer rowLimitOverride, + Long bytesScannedLimitOverride, Instant expiresAt) { + + /** Backward-compatible constructor without the #941 bytes-scanned cap override. */ + public DatasourceUserPermissionView(UUID id, UUID userId, UUID datasourceId, boolean canRead, + boolean canWrite, boolean canDdl, boolean canBreakGlass, + List allowedSchemas, List allowedTables, + List restrictedColumns, List deniedColumns, + List deniedSchemas, List deniedTables, + List deniedShapes, Integer rowLimitOverride, + Instant expiresAt) { + this(id, userId, datasourceId, canRead, canWrite, canDdl, canBreakGlass, allowedSchemas, + allowedTables, restrictedColumns, deniedColumns, deniedSchemas, deniedTables, + deniedShapes, rowLimitOverride, null, expiresAt); + } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java index b579fd559..a7ef06597 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/DatasourceView.java @@ -31,7 +31,9 @@ public record DatasourceView( String localDatacenter, boolean resultCacheEnabled, Integer resultCacheTtlSeconds, - DatasourceEnvironment environment + DatasourceEnvironment environment, + Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { /** One read-replica endpoint as exposed to admins — never carries the password. */ public record ReadReplicaView(UUID id, String jdbcUrl, String username) { @@ -39,6 +41,28 @@ public record ReadReplicaView(UUID id, String jdbcUrl, String username) { public DatasourceView { readReplicas = readReplicas == null ? List.of() : List.copyOf(readReplicas); + bytesCapMissingEstimate = bytesCapMissingEstimate == null + ? BytesCapMissingEstimateAction.REQUIRE_REVIEW : bytesCapMissingEstimate; + } + + /** + * Backward-compatible constructor for the pre-#941 canonical shape (no bytes-scanned cap); + * delegates with no cap and the default missing-estimate policy. + */ + public DatasourceView( + UUID id, UUID organizationId, String name, DbType dbType, String host, Integer port, + String databaseName, String username, SslMode sslMode, int connectionPoolSize, + int maxRowsPerQuery, boolean requireReviewReads, boolean requireReviewWrites, + UUID reviewPlanId, boolean aiAnalysisEnabled, UUID aiConfigId, boolean textToSqlEnabled, + UUID customDriverId, String connectorId, String jdbcUrlOverride, + List readReplicas, boolean active, Instant createdAt, + String localDatacenter, boolean resultCacheEnabled, Integer resultCacheTtlSeconds, + DatasourceEnvironment environment) { + this(id, organizationId, name, dbType, host, port, databaseName, username, sslMode, + connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, + reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, + connectorId, jdbcUrlOverride, readReplicas, active, createdAt, localDatacenter, + resultCacheEnabled, resultCacheTtlSeconds, environment, null, null); } /** @@ -57,7 +81,7 @@ public DatasourceView( connectionPoolSize, maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, customDriverId, connectorId, jdbcUrlOverride, readReplicas, active, createdAt, localDatacenter, - resultCacheEnabled, resultCacheTtlSeconds, null); + resultCacheEnabled, resultCacheTtlSeconds, (DatasourceEnvironment) null); } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java index 615396324..98851f9bb 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/PersistQueryEstimateCommand.java @@ -3,7 +3,8 @@ /** * Everything needed to persist one {@code query_estimates} row (issue AF-624). Mirrors * {@link PersistAiAnalysisCommand}: the proxy module computes the estimate and hands the values to - * {@link QueryEstimatePersistenceService}. All value fields are nullable best-effort signals. + * {@link QueryEstimatePersistenceService}. All value fields are nullable best-effort signals; + * {@code estimatedBytesScanned} is the warehouse engines' native scan estimate in raw bytes (#941). */ public record PersistQueryEstimateCommand( String engineId, @@ -13,6 +14,7 @@ public record PersistQueryEstimateCommand( Long affectedRowCount, String scanType, Double estimatedCost, + Long estimatedBytesScanned, String planJson, String rawPlan, String unsupportedReason, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java index 9f799408b..163b18db8 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryDetailView.java @@ -49,7 +49,40 @@ public record QueryDetailView( String onBehalfOfEmail, /** The calling application (#938) and how it was learned; both null when unknown. */ String applicationName, - ApplicationNameSource applicationNameSource) { + ApplicationNameSource applicationNameSource, + /** The bytes-scanned cap (#941) that applied when the query left PENDING_AI; null when none. */ + BytesScannedCapDetail bytesScannedCap) { + + /** Backward-compatible constructor without the #941 bytes-scanned cap. */ + public QueryDetailView(UUID id, UUID datasourceId, String datasourceName, DbType dbType, + UUID organizationId, UUID submittedByUserId, String submittedByEmail, + String submittedByDisplayName, String sqlText, QueryType queryType, + QueryStatus status, String justification, AiAnalysisDetail aiAnalysis, + CostEstimateDetail costEstimate, + ApprovalPredictionDetail approvalPrediction, Long rowsAffected, + Integer durationMs, String errorMessage, UUID previousRunId, + UUID approvedByGrantId, String reviewPlanName, + Integer approvalTimeoutHours, Instant escalatedAt, + Integer escalationAfterHours, List reviewDecisions, + Instant scheduledFor, String recurrenceRule, Instant recurrenceUntil, + Instant recurrenceNextRunAt, String recurrenceHaltedReason, + UUID recurringParentId, Instant createdAt, Instant updatedAt, + UUID onBehalfOfUserId, String onBehalfOfEmail, String applicationName, + ApplicationNameSource applicationNameSource) { + this(id, datasourceId, datasourceName, dbType, organizationId, submittedByUserId, + submittedByEmail, submittedByDisplayName, sqlText, queryType, status, justification, + aiAnalysis, costEstimate, approvalPrediction, rowsAffected, durationMs, + errorMessage, previousRunId, approvedByGrantId, reviewPlanName, + approvalTimeoutHours, escalatedAt, escalationAfterHours, reviewDecisions, + scheduledFor, recurrenceRule, recurrenceUntil, recurrenceNextRunAt, + recurrenceHaltedReason, recurringParentId, createdAt, updatedAt, onBehalfOfUserId, + onBehalfOfEmail, applicationName, applicationNameSource, null); + } + + /** The bytes-scanned cap (#941) recorded on the request, and how its estimate compared. */ + public record BytesScannedCapDetail(long limit, BytesScannedCapSource source, + BytesScannedCapOutcome outcome) { + } /** Backward-compatible constructor without the #938 calling application. */ public QueryDetailView(UUID id, UUID datasourceId, String datasourceName, DbType dbType, @@ -73,7 +106,7 @@ public QueryDetailView(UUID id, UUID datasourceId, String datasourceName, DbType approvalTimeoutHours, escalatedAt, escalationAfterHours, reviewDecisions, scheduledFor, recurrenceRule, recurrenceUntil, recurrenceNextRunAt, recurrenceHaltedReason, recurringParentId, createdAt, updatedAt, onBehalfOfUserId, - onBehalfOfEmail, null, null); + onBehalfOfEmail, (String) null, (ApplicationNameSource) null); } /** Backward-compatible constructor without the #874 on-behalf-of principal. */ @@ -186,7 +219,20 @@ public record CostEstimateDetail( String unsupportedReason, boolean failed, String errorMessage, - Integer durationMs) { + Integer durationMs, + /** The warehouse engines' pre-flight scan estimate in raw bytes (#941); else null. */ + Long estimatedBytesScanned) { + + /** Backward-compatible constructor without the #941 bytes estimate. */ + public CostEstimateDetail(UUID id, String engineId, QueryType queryType, boolean supported, + Long estimatedRows, Long affectedRowCount, String scanType, + Double estimatedCost, String planJson, String rawPlan, + String unsupportedReason, boolean failed, String errorMessage, + Integer durationMs) { + this(id, engineId, queryType, supported, estimatedRows, affectedRowCount, scanType, + estimatedCost, planJson, rawPlan, unsupportedReason, failed, errorMessage, + durationMs, null); + } } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java index bd118dddb..9d282baf5 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryEstimateSnapshot.java @@ -10,7 +10,8 @@ * is {@code false} when the engine has no plan concept; {@code failed} marks an unexpected * computation error. {@code estimatedRows}, {@code affectedRowCount}, {@code scanType} and * {@code estimatedCost} are individually nullable — best-effort signals filled from what the - * engine's plan exposes. + * engine's plan exposes. {@code estimatedBytesScanned} (#941) is the warehouse engines' native + * pre-flight scan estimate in raw bytes, null for every engine that does not report one. */ public record QueryEstimateSnapshot( UUID id, @@ -22,6 +23,7 @@ public record QueryEstimateSnapshot( Long affectedRowCount, String scanType, Double estimatedCost, + Long estimatedBytesScanned, String planJson, String rawPlan, String unsupportedReason, diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java index 690bde05a..826e2d896 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/QueryRequestStateService.java @@ -25,6 +25,14 @@ public interface QueryRequestStateService { */ void approveByAccessGrant(UUID queryRequestId, UUID accessGrantId); + /** + * Records the bytes-scanned cap (#941) that applied when the query left {@code PENDING_AI}, + * and how its estimate compared. A plain stamp, not a transition — it runs just before the + * decision is applied so the detail view can explain the outcome. + */ + void recordBytesScannedCap(UUID queryRequestId, long limit, BytesScannedCapSource source, + BytesScannedCapOutcome outcome); + /** * Inserts an {@code APPROVED} {@link com.bablsoft.accessflow.core.api.DecisionType} row * for the given reviewer/stage and, if the per-stage threshold is now met AND it was the diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java b/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java index 020fa65b2..ece064003 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/api/UpdateDatasourceCommand.java @@ -10,7 +10,8 @@ * {@code readReplicas} (AF-457), {@code null} keeps the current endpoint list, an empty list * deletes all endpoints, and a non-empty list is a full replacement merged by endpoint id (items * with a known {@code id} update that row; items without create new rows; stored rows absent from - * the list are removed). + * the list are removed). The bytes-scanned cap (#941) follows the same shape as the environment: + * {@code clearMaxBytesScannedPerQuery=true} removes the cap, a non-null value wins when both are sent. */ public record UpdateDatasourceCommand( String name, @@ -38,8 +39,33 @@ public record UpdateDatasourceCommand( Integer resultCacheTtlSeconds, String privateKeyPassphrase, DatasourceEnvironment environment, - Boolean clearEnvironment + Boolean clearEnvironment, + Long maxBytesScannedPerQuery, + Boolean clearMaxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { + /** + * Backward-compatible constructor for the pre-#941 canonical shape (no bytes-scanned cap); + * delegates with {@code null} — the cap and its policy stay unchanged. + */ + public UpdateDatasourceCommand( + String name, String host, Integer port, String databaseName, String username, + String password, SslMode sslMode, Integer connectionPoolSize, Integer maxRowsPerQuery, + Boolean requireReviewReads, Boolean requireReviewWrites, UUID reviewPlanId, + Boolean aiAnalysisEnabled, UUID aiConfigId, Boolean textToSqlEnabled, + Boolean clearAiConfig, String jdbcUrlOverride, + List readReplicas, Boolean active, String localDatacenter, + String apiKey, Boolean resultCacheEnabled, Integer resultCacheTtlSeconds, + String privateKeyPassphrase, DatasourceEnvironment environment, + Boolean clearEnvironment) { + this(name, host, port, databaseName, username, password, sslMode, connectionPoolSize, + maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, + aiAnalysisEnabled, aiConfigId, textToSqlEnabled, clearAiConfig, jdbcUrlOverride, + readReplicas, active, localDatacenter, apiKey, resultCacheEnabled, + resultCacheTtlSeconds, privateKeyPassphrase, environment, clearEnvironment, + null, null, null); + } + /** * Backward-compatible constructor for the pre-#861 canonical shape (no {@code environment} / * {@code clearEnvironment}); delegates with {@code null} — the environment stays unchanged. @@ -57,7 +83,8 @@ public UpdateDatasourceCommand( maxRowsPerQuery, requireReviewReads, requireReviewWrites, reviewPlanId, aiAnalysisEnabled, aiConfigId, textToSqlEnabled, clearAiConfig, jdbcUrlOverride, readReplicas, active, localDatacenter, apiKey, resultCacheEnabled, - resultCacheTtlSeconds, privateKeyPassphrase, null, null); + resultCacheTtlSeconds, privateKeyPassphrase, (DatasourceEnvironment) null, + (Boolean) null); } /** diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java b/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java index bc0d8cbf3..5da8a9581 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/events/QueryAutoRejectedEvent.java @@ -3,9 +3,10 @@ import java.util.UUID; /** - * Published when a query is auto-rejected by the state machine because a routing policy matched with - * the {@code AUTO_REJECT} action. Carries the matched {@code routing_policy.id} and the reason so the - * audit and notification listeners can record provenance. + * Published when a query is rejected without a reviewer: a routing policy matched with the + * {@code AUTO_REJECT} action (carrying its {@code routing_policy.id}), the bytes-scanned cap refused + * it (#941), or an external ticket resolution rejected it — the last two with a null policy id. The + * reason lets the audit and notification listeners record provenance. */ public record QueryAutoRejectedEvent(UUID queryRequestId, UUID matchedPolicyId, String reason) { } diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java index bfb9f64b4..81aa1dd9d 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImpl.java @@ -21,6 +21,8 @@ import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.DeniedColumns; import com.bablsoft.accessflow.core.api.DeniedColumnsNotSupportedException; +import com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException; +import com.bablsoft.accessflow.core.api.BytesScannedCapSupport; import com.bablsoft.accessflow.core.api.DeniedShapes; import com.bablsoft.accessflow.core.api.DeniedShapesNotSupportedException; import com.bablsoft.accessflow.core.api.DeniedTables; @@ -249,6 +251,11 @@ public DatasourceView create(CreateDatasourceCommand command) { entity.setResultCacheTtlSeconds(command.resultCacheTtlSeconds()); } entity.setEnvironment(command.environment()); + entity.setMaxBytesScannedPerQuery(bytesCap(entity.getDbType(), + command.maxBytesScannedPerQuery())); + if (command.bytesCapMissingEstimate() != null) { + entity.setBytesCapMissingEstimate(command.bytesCapMissingEstimate()); + } entity.setActive(true); return toView(datasourceRepository.save(entity)); } @@ -360,6 +367,17 @@ public DatasourceView update(UUID id, UUID organizationId, UpdateDatasourceComma if (command.environment() != null) { entity.setEnvironment(command.environment()); } + // #941: same null / clear / value shape as the environment. + if (Boolean.TRUE.equals(command.clearMaxBytesScannedPerQuery())) { + entity.setMaxBytesScannedPerQuery(null); + } + if (command.maxBytesScannedPerQuery() != null) { + entity.setMaxBytesScannedPerQuery(bytesCap(entity.getDbType(), + command.maxBytesScannedPerQuery())); + } + if (command.bytesCapMissingEstimate() != null) { + entity.setBytesCapMissingEstimate(command.bytesCapMissingEstimate()); + } if (command.active() != null) { entity.setActive(command.active()); } @@ -654,6 +672,8 @@ public DatasourcePermissionView grantPermission(UUID datasourceId, UUID organiza entity.setCanDdl(Boolean.TRUE.equals(command.canDdl())); entity.setCanBreakGlass(Boolean.TRUE.equals(command.canBreakGlass())); entity.setRowLimitOverride(command.rowLimitOverride()); + entity.setBytesScannedLimitOverride(bytesCap(datasource.getDbType(), + command.bytesScannedLimitOverride())); entity.setAllowedSchemas(toArray(command.allowedSchemas())); entity.setAllowedTables(toArray(command.allowedTables())); entity.setRestrictedColumns(toArray(command.restrictedColumns())); @@ -714,6 +734,8 @@ public DatasourceGroupPermissionView grantGroupPermission( entity.setCanDdl(Boolean.TRUE.equals(command.canDdl())); entity.setCanBreakGlass(Boolean.TRUE.equals(command.canBreakGlass())); entity.setRowLimitOverride(command.rowLimitOverride()); + entity.setBytesScannedLimitOverride(bytesCap(datasource.getDbType(), + command.bytesScannedLimitOverride())); entity.setAllowedSchemas(toArray(command.allowedSchemas())); entity.setAllowedTables(toArray(command.allowedTables())); entity.setRestrictedColumns(toArray(command.restrictedColumns())); @@ -780,7 +802,9 @@ private DatasourceView toView(DatasourceEntity entity) { entity.getLocalDatacenter(), entity.isResultCacheEnabled(), entity.getResultCacheTtlSeconds(), - entity.getEnvironment()); + entity.getEnvironment(), + entity.getMaxBytesScannedPerQuery(), + entity.getBytesCapMissingEstimate()); } private CustomJdbcDriverEntity resolveCustomDriverForCreate(CreateDatasourceCommand command) { @@ -1150,6 +1174,17 @@ private String[] deniedShapes(DatasourceEntity datasource, List raw) return names; } + /** + * A bytes-scanned cap (#941) is refused on an engine that reports no bytes estimate — there it + * could only reject everything or nothing. Null passes through (no cap). + */ + private static Long bytesCap(DbType dbType, Long cap) { + if (cap != null && !BytesScannedCapSupport.supports(dbType)) { + throw new BytesScannedCapNotSupportedException(dbType); + } + return cap; + } + private DatasourcePermissionView toPermissionView(DatasourceUserPermissionEntity entity) { UserEntity user = entity.getUser(); return new DatasourcePermissionView( @@ -1163,6 +1198,7 @@ private DatasourcePermissionView toPermissionView(DatasourceUserPermissionEntity entity.isCanDdl(), entity.isCanBreakGlass(), entity.getRowLimitOverride(), + entity.getBytesScannedLimitOverride(), toList(entity.getAllowedSchemas()), toList(entity.getAllowedTables()), toList(entity.getRestrictedColumns()), @@ -1189,6 +1225,7 @@ private DatasourceGroupPermissionView toGroupPermissionView( entity.isCanDdl(), entity.isCanBreakGlass(), entity.getRowLimitOverride(), + entity.getBytesScannedLimitOverride(), toList(entity.getAllowedSchemas()), toList(entity.getAllowedTables()), toList(entity.getRestrictedColumns()), diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java new file mode 100644 index 000000000..c069a6cf5 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionService.java @@ -0,0 +1,49 @@ +package com.bablsoft.accessflow.core.internal; + +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionLookupService; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionView; +import com.bablsoft.accessflow.core.internal.persistence.repo.DatasourceRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Optional; +import java.util.UUID; + +/** + * The most restrictive of the datasource's {@code max_bytes_scanned_per_query} and the user's merged + * grant override (#941). On a tie the datasource is named as the source — it is the setting an + * admin would have to change for the limit to move. + */ +@Service +@RequiredArgsConstructor +class DefaultBytesScannedCapResolutionService implements BytesScannedCapResolutionService { + + private final DatasourceRepository datasourceRepository; + private final DatasourceUserPermissionLookupService permissionLookupService; + + @Override + @Transactional(readOnly = true) + public Optional resolve(UUID datasourceId, UUID userId) { + var datasource = datasourceRepository.findById(datasourceId).orElse(null); + if (datasource == null) { + return Optional.empty(); + } + var datasourceCap = datasource.getMaxBytesScannedPerQuery(); + var grantCap = userId == null ? null : permissionLookupService.findFor(userId, datasourceId) + .map(DatasourceUserPermissionView::bytesScannedLimitOverride) + .orElse(null); + if (datasourceCap == null && grantCap == null) { + return Optional.empty(); + } + var missing = datasource.getBytesCapMissingEstimate(); + if (grantCap != null && (datasourceCap == null || grantCap < datasourceCap)) { + return Optional.of(new AppliedBytesCap(grantCap, BytesScannedCapSource.GRANT, missing)); + } + return Optional.of(new AppliedBytesCap(datasourceCap, BytesScannedCapSource.DATASOURCE, + missing)); + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java index d2864909b..e0100f78d 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupService.java @@ -215,6 +215,7 @@ private static DatasourceUserPermissionView merge(UUID userId, UUID datasourceId unionDenied(parts, DatasourcePermissionContribution::deniedTables), unionDeniedShapes(parts), minRowLimit(parts), + minBytesLimit(parts), anyNeverExpires ? null : expiresAt); } @@ -254,6 +255,18 @@ private static Integer minRowLimit(List parts) return min; } + /** Like {@link #minRowLimit}: the most restrictive bytes-scanned cap wins (#941). */ + private static Long minBytesLimit(List parts) { + Long min = null; + for (var p : parts) { + var limit = p.bytesScannedLimitOverride(); + if (limit != null && (min == null || limit < min)) { + min = limit; + } + } + return min; + } + /** Allow-list union: a null/empty contribution means "all allowed", so it wins → empty list. */ private static List unionAllowList( List parts, @@ -318,6 +331,7 @@ private static DatasourceUserPermissionView toDirectView(DatasourceUserPermissio DeniedTables.normalize(toList(entity.getDeniedTables())), DeniedShapes.fromNames(toList(entity.getDeniedShapes())), entity.getRowLimitOverride(), + entity.getBytesScannedLimitOverride(), entity.getExpiresAt()); } @@ -330,7 +344,8 @@ private static DatasourcePermissionContribution toContribution( toList(e.getRestrictedColumns()), toList(e.getDeniedColumns()), toList(e.getDeniedSchemas()), toList(e.getDeniedTables()), DeniedShapes.fromNames(toList(e.getDeniedShapes())), - e.getRowLimitOverride(), e.getExpiresAt(), e.getAccessGrantRequestId()); + e.getRowLimitOverride(), e.getBytesScannedLimitOverride(), e.getExpiresAt(), + e.getAccessGrantRequestId()); } private static DatasourcePermissionContribution toContribution( @@ -342,7 +357,8 @@ private static DatasourcePermissionContribution toContribution( toList(e.getRestrictedColumns()), toList(e.getDeniedColumns()), toList(e.getDeniedSchemas()), toList(e.getDeniedTables()), DeniedShapes.fromNames(toList(e.getDeniedShapes())), - e.getRowLimitOverride(), e.getExpiresAt(), null); + e.getRowLimitOverride(), e.getBytesScannedLimitOverride(), e.getExpiresAt(), + null); } private static List toList(String[] array) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java index 642303f49..40c809d5f 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateService.java @@ -43,6 +43,7 @@ public UUID persist(UUID queryRequestId, PersistQueryEstimateCommand command) { entity.setAffectedRowCount(command.affectedRowCount()); entity.setScanType(command.scanType()); entity.setEstimatedCost(command.estimatedCost()); + entity.setEstimatedBytesScanned(command.estimatedBytesScanned()); entity.setPlan(command.planJson()); entity.setRawPlan(command.rawPlan()); entity.setUnsupportedReason(command.unsupportedReason()); @@ -72,6 +73,7 @@ private QueryEstimateSnapshot toSnapshot(QueryEstimateEntity entity) { entity.getAffectedRowCount(), entity.getScanType(), entity.getEstimatedCost(), + entity.getEstimatedBytesScanned(), entity.getPlan(), entity.getRawPlan(), entity.getUnsupportedReason(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java index 5c1892425..3b2271e1f 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupService.java @@ -362,7 +362,13 @@ private QueryDetailView toDetailView(QueryRequestEntity entity) { entity.getOnBehalfOfUserId(), onBehalfOfEmail(entity.getOnBehalfOfUserId()), entity.getApplicationName(), - entity.getApplicationNameSource()); + entity.getApplicationNameSource(), + entity.getBytesScannedCap() != null && entity.getBytesScannedCapSource() != null + && entity.getBytesScannedCapOutcome() != null + ? new QueryDetailView.BytesScannedCapDetail(entity.getBytesScannedCap(), + entity.getBytesScannedCapSource(), + entity.getBytesScannedCapOutcome()) + : null); } private String onBehalfOfEmail(UUID onBehalfOfUserId) { @@ -444,7 +450,8 @@ private static QueryDetailView.CostEstimateDetail toCostEstimateDetail( entity.getUnsupportedReason(), entity.isFailed(), entity.getErrorMessage(), - entity.getDurationMs()); + entity.getDurationMs(), + entity.getEstimatedBytesScanned()); } private static QueryDetailView.ApprovalPredictionDetail toApprovalPredictionDetail( diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java index 2f47c6be0..30788db44 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateService.java @@ -3,6 +3,8 @@ import com.bablsoft.accessflow.core.api.DecisionType; import com.bablsoft.accessflow.core.api.IllegalQueryStatusTransitionException; import com.bablsoft.accessflow.core.api.QueryRequestNotFoundException; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.QueryRequestStateService; import com.bablsoft.accessflow.core.api.QueryStatus; import com.bablsoft.accessflow.core.api.RecordApprovalCommand; @@ -65,6 +67,18 @@ public void approveByAccessGrant(UUID queryRequestId, UUID accessGrantId) { publishStatusChanged(entity, previous, QueryStatus.APPROVED); } + @Override + @Transactional + public void recordBytesScannedCap(UUID queryRequestId, long limit, + BytesScannedCapSource source, + BytesScannedCapOutcome outcome) { + var entity = lockOrThrow(queryRequestId); + entity.setBytesScannedCap(limit); + entity.setBytesScannedCapSource(source); + entity.setBytesScannedCapOutcome(outcome); + queryRequestRepository.save(entity); + } + @Override @Transactional public RecordDecisionResult recordApprovalAndAdvance(RecordApprovalCommand command) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java index c68d54602..6f4172fd3 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceEntity.java @@ -1,6 +1,7 @@ package com.bablsoft.accessflow.core.internal.persistence.entity; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.SslMode; @@ -124,6 +125,17 @@ public class DatasourceEntity { @Column(name = "environment", columnDefinition = "datasource_environment") private DatasourceEnvironment environment; + /** #941: pre-flight bytes-scanned cap; null = no cap. Only set on bytes-reporting engines. */ + @Column(name = "max_bytes_scanned_per_query") + private Long maxBytesScannedPerQuery; + + @Enumerated(EnumType.STRING) + @JdbcType(PostgreSQLEnumJdbcType.class) + @Column(name = "bytes_cap_missing_estimate", nullable = false, + columnDefinition = "bytes_cap_missing_estimate_action") + private BytesCapMissingEstimateAction bytesCapMissingEstimate = + BytesCapMissingEstimateAction.REQUIRE_REVIEW; + @JsonIgnore @Column(name = "api_key_encrypted", columnDefinition = "TEXT") private String apiKeyEncrypted; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java index 4c80ac2a0..0cedcdf81 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceGroupPermissionEntity.java @@ -53,6 +53,10 @@ public class DatasourceGroupPermissionEntity { @Column(name = "row_limit_override") private Integer rowLimitOverride; + /** #941: this grant's bytes-scanned cap; null = none. Most restrictive across grants wins. */ + @Column(name = "bytes_scanned_limit_override") + private Long bytesScannedLimitOverride; + @JdbcTypeCode(SqlTypes.ARRAY) @Column(name = "allowed_schemas", columnDefinition = "text[]") private String[] allowedSchemas; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java index d2393fd2c..a9c4c8647 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/DatasourceUserPermissionEntity.java @@ -49,6 +49,10 @@ public class DatasourceUserPermissionEntity { @Column(name = "row_limit_override") private Integer rowLimitOverride; + /** #941: this grant's bytes-scanned cap; null = none. Most restrictive across grants wins. */ + @Column(name = "bytes_scanned_limit_override") + private Long bytesScannedLimitOverride; + @JdbcTypeCode(SqlTypes.ARRAY) @Column(name = "allowed_schemas", columnDefinition = "text[]") private String[] allowedSchemas; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java index c59b8f1aa..fd73f6945 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryEstimateEntity.java @@ -58,6 +58,9 @@ public class QueryEstimateEntity { @Column(name = "estimated_cost") private Double estimatedCost; + @Column(name = "estimated_bytes_scanned") + private Long estimatedBytesScanned; + @JdbcTypeCode(SqlTypes.JSON) @Column(columnDefinition = "jsonb") private String plan; diff --git a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java index 1db2d7440..73d3c690e 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java +++ b/backend/src/main/java/com/bablsoft/accessflow/core/internal/persistence/entity/QueryRequestEntity.java @@ -1,6 +1,8 @@ package com.bablsoft.accessflow.core.internal.persistence.entity; import com.bablsoft.accessflow.core.api.ApplicationNameSource; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.QueryStatus; import com.bablsoft.accessflow.core.api.QueryType; import com.bablsoft.accessflow.core.api.SubmissionReason; @@ -152,6 +154,21 @@ public class QueryRequestEntity { @Column(name = "application_name_source", columnDefinition = "application_name_source") private ApplicationNameSource applicationNameSource; + // The bytes-scanned cap (#941) that applied when the query left PENDING_AI — stamped by the + // workflow so the detail view can say why a query was refused or forced to review. + @Column(name = "bytes_scanned_cap") + private Long bytesScannedCap; + + @Enumerated(EnumType.STRING) + @JdbcType(PostgreSQLEnumJdbcType.class) + @Column(name = "bytes_scanned_cap_source", columnDefinition = "bytes_scanned_cap_source") + private BytesScannedCapSource bytesScannedCapSource; + + @Enumerated(EnumType.STRING) + @JdbcType(PostgreSQLEnumJdbcType.class) + @Column(name = "bytes_scanned_cap_outcome", columnDefinition = "bytes_scanned_cap_outcome") + private BytesScannedCapOutcome bytesScannedCapOutcome; + @Version @Column(name = "updated_at", nullable = false) private Instant updatedAt = Instant.now(); diff --git a/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java b/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java index e93c831d5..7ae79fe13 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/proxy/api/QueryCostEstimateService.java @@ -1,6 +1,7 @@ package com.bablsoft.accessflow.proxy.api; import com.bablsoft.accessflow.core.api.QueryEstimateSnapshot; +import com.bablsoft.accessflow.core.api.QueryExecutionRequest; import java.util.Optional; import java.util.UUID; @@ -22,4 +23,13 @@ public interface QueryCostEstimateService { * caller. Empty only when the query request itself does not exist. */ Optional estimateSubmittedQuery(UUID queryRequestId); + + /** + * The warehouse's pre-flight bytes-scanned estimate for a statement that has no persisted + * estimate of its own (#941 — a request-group member), dry-run under the same + * {@code accessflow.proxy.estimate-timeout} as the submission estimate. Nothing is persisted. + * Empty when the engine reports no bytes figure, the plan is unsupported, or the dry-run fails — + * never throws. + */ + Optional estimateBytesScanned(QueryExecutionRequest request); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java b/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java index aebd2a74a..62d4e5e97 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateService.java @@ -84,7 +84,7 @@ public Optional estimateSubmittedQuery(UUID queryRequestI } catch (RuntimeException ex) { log.warn("Cost estimate failed for query {}: {}", queryRequestId, ex.getMessage()); var command = new PersistQueryEstimateCommand(null, snapshot.queryType(), false, null, - null, null, null, null, null, null, true, truncate(ex.getMessage()), + null, null, null, null, null, null, null, true, truncate(ex.getMessage()), durationMs(start)); var result = persistAndPublish(queryRequestId, command, false); eventPublisher.publishEvent( @@ -93,6 +93,25 @@ null, null, null, null, null, null, true, truncate(ex.getMessage()), } } + @Override + public Optional estimateBytesScanned(QueryExecutionRequest request) { + var bounded = new QueryExecutionRequest(request.datasourceId(), request.sql(), + request.queryType(), request.maxRowsOverride(), properties.estimateTimeout(), + request.restrictedColumns(), request.columnMasks(), + request.rowSecurityPredicates(), request.transactional(), request.statements(), + request.softDeleteDirectives(), request.referencedTables()); + try { + var dryRun = queryExecutor.dryRun(bounded); + return dryRun.supported() + ? Optional.ofNullable(dryRun.estimatedBytesScanned()) + : Optional.empty(); + } catch (RuntimeException ex) { + log.debug("Bytes estimate failed for datasource {}: {}", request.datasourceId(), + ex.getMessage()); + return Optional.empty(); + } + } + private QueryExecutionRequest buildRequest(QueryRequestSnapshot snapshot) { var rowSecurityPredicates = rowSecurityResolutionService .resolveApplicable(snapshot.organizationId(), snapshot.datasourceId(), @@ -144,6 +163,7 @@ private PersistQueryEstimateCommand toCommand(QueryRequestSnapshot snapshot, affectedRows, access != null ? truncateTo(access.operation(), 128) : null, root != null ? root.estimatedCost() : null, + dryRun.estimatedBytesScanned(), planJson(root, redactPredicates), redactPredicates ? null : dryRun.rawPlan(), null, false, null, durationMs); } @@ -167,14 +187,14 @@ private static QueryPlanNode accessNode(QueryPlanNode root, QueryType queryType) private PersistQueryEstimateCommand unsupportedCommand(QueryRequestSnapshot snapshot, String engineId, String reason) { return new PersistQueryEstimateCommand(engineId, snapshot.queryType(), false, null, null, - null, null, null, null, truncate(reason), false, null, null); + null, null, null, null, null, truncate(reason), false, null, null); } private static PersistQueryEstimateCommand withAffectedRows(PersistQueryEstimateCommand command, Long affectedRows, int durationMs) { return new PersistQueryEstimateCommand(command.engineId(), command.queryType(), command.supported(), command.estimatedRows(), affectedRows, command.scanType(), - command.estimatedCost(), command.planJson(), command.rawPlan(), + command.estimatedCost(), command.estimatedBytesScanned(), command.planJson(), command.rawPlan(), command.unsupportedReason(), command.failed(), command.errorMessage(), durationMs); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java index 6227e9e31..aec847e09 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionService.java @@ -6,6 +6,9 @@ import com.bablsoft.accessflow.audit.api.AuditEntry; import com.bablsoft.accessflow.audit.api.AuditLogService; import com.bablsoft.accessflow.audit.api.AuditResourceType; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; +import com.bablsoft.accessflow.core.api.BytesScannedCapExceededException; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; import com.bablsoft.accessflow.core.api.ColumnMaskDirective; import com.bablsoft.accessflow.core.api.DatasourceLookupService; import com.bablsoft.accessflow.core.api.DatasourceUserPermissionLookupService; @@ -17,7 +20,11 @@ import com.bablsoft.accessflow.core.api.RowSecurityResolutionService; import com.bablsoft.accessflow.core.api.SelectExecutionResult; import com.bablsoft.accessflow.core.api.UpdateExecutionResult; +import com.bablsoft.accessflow.proxy.api.QueryCostEstimateService; import com.bablsoft.accessflow.proxy.api.QueryExecutor; +import com.bablsoft.accessflow.requestgroups.internal.persistence.repo.GroupReviewDecisionRepository; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.DecisionType; import com.bablsoft.accessflow.proxy.api.QueryParser; import com.bablsoft.accessflow.requestgroups.api.RequestGroupItemStatus; import com.bablsoft.accessflow.requestgroups.api.RequestGroupStatus; @@ -31,9 +38,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.MessageSource; +import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Service; import java.time.Instant; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -66,6 +76,10 @@ public class GroupExecutionService { private final ApiInlineExecutionService apiInlineExecutionService; private final AuditLogService auditLogService; private final ApplicationEventPublisher eventPublisher; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; + private final MessageSource messageSource; + private final QueryCostEstimateService queryCostEstimateService; + private final GroupReviewDecisionRepository decisionRepository; /** Execute an APPROVED group. Idempotent: silently returns if it is not APPROVED (or not yet due). */ public void execute(UUID groupId, UUID actorUserId, String trigger) { @@ -172,10 +186,12 @@ private void runQuery(RequestGroupEntity group, RequestGroupItemEntity item) { if (appliedRowLimit.isPresent()) { rowLimitOverride = appliedRowLimit.get().tighten(rowLimitOverride); } - var result = queryExecutor.execute(new QueryExecutionRequest( + var request = new QueryExecutionRequest( item.getDatasourceId(), item.getSqlText(), item.getQueryType(), rowLimitOverride, null, restrictedColumns, columnMasks, rowSecurity, parsed.transactional(), - parsed.statements(), List.of(), parsed.referencedTables())); + parsed.statements(), List.of(), parsed.referencedTables()); + enforceBytesScannedCap(group, item, request); + var result = queryExecutor.execute(request); long rows = switch (result) { case SelectExecutionResult select -> select.rowCount(); case UpdateExecutionResult update -> update.rowsAffected(); @@ -220,6 +236,59 @@ private void auditMember(RequestGroupEntity group, RequestGroupItemEntity item, "member_status", item.getStatus().name())); } + /** + * A member has no persisted estimate of its own, so when a bytes-scanned cap (#941) binds the + * submitter the member is dry-run here — only then, so an uncapped group pays nothing. A + * refusal fails the member like any execution error ({@code continue_on_error} decides the + * rest). A missing estimate under {@code REQUIRE_REVIEW} passes only when a person approved + * the group. + */ + private void enforceBytesScannedCap(RequestGroupEntity group, RequestGroupItemEntity item, + QueryExecutionRequest request) { + var cap = bytesScannedCapResolutionService + .resolve(item.getDatasourceId(), group.getSubmittedBy()) + .orElse(null); + if (cap == null) { + return; + } + var estimated = queryCostEstimateService.estimateBytesScanned(request).orElse(null); + var outcome = cap.check(estimated); + // A group can be auto-approved by its plans without anyone looking at it, and the cap is + // not consulted at that point — so "require review" is only satisfied when a person + // actually approved the group. Otherwise the member is refused rather than run unreviewed. + boolean unreviewed = outcome == BytesScannedCapOutcome.NO_ESTIMATE_REVIEW + && !decisionRepository.existsByRequestGroupIdAndDecision(group.getId(), + DecisionType.APPROVED); + if (!outcome.rejects() && !unreviewed) { + return; + } + var limit = ByteSizeFormat.format(cap.limit()); + var locale = LocaleContextHolder.getLocale(); + String message; + if (estimated != null) { + message = messageSource.getMessage("error.bytes_cap.exceeded", + new Object[]{ByteSizeFormat.format(estimated), limit}, locale); + } else if (unreviewed) { + message = messageSource.getMessage("error.bytes_cap.no_estimate_unreviewed", + new Object[]{limit}, locale); + } else { + message = messageSource.getMessage("error.bytes_cap.no_estimate", new Object[]{limit}, + locale); + } + var metadata = new HashMap(); + metadata.put("trigger", "bytes_scanned_cap"); + metadata.put("stage", "execution"); + metadata.put("item_id", item.getId().toString()); + metadata.put("limit", cap.limit()); + metadata.put("source", cap.source().name()); + if (estimated != null) { + metadata.put("estimated_bytes", estimated); + } + metadata.put("outcome", outcome.name()); + audit(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, group, null, metadata); + throw new BytesScannedCapExceededException(message, cap, estimated, outcome); + } + private void audit(AuditAction action, RequestGroupEntity group, UUID actorId, Map metadata) { try { diff --git a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java index 1c6ab3e48..2fd025c84 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java +++ b/backend/src/main/java/com/bablsoft/accessflow/requestgroups/internal/persistence/repo/GroupReviewDecisionRepository.java @@ -18,5 +18,8 @@ public interface GroupReviewDecisionRepository extends JpaRepository findByRequestGroupIdAndReviewerIdAndStage( UUID requestGroupId, UUID reviewerId, int stage); + /** Whether any person approved the group — the review a missing bytes estimate demands (#941). */ + boolean existsByRequestGroupIdAndDecision(UUID requestGroupId, DecisionType decision); + long countByRequestGroupIdAndStageAndDecision(UUID requestGroupId, int stage, DecisionType decision); } diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java index 87dabc4c6..8e45ae795 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/DatasourceController.java @@ -143,7 +143,7 @@ DatasourcePageResponse listDatasources(Authentication authentication, Pageable p @ApiResponse(responseCode = "201", description = "Datasource created") @ApiResponse(responseCode = "400", description = "Validation error") @ApiResponse(responseCode = "409", description = "A datasource with this name already exists") - @ApiResponse(responseCode = "422", description = "JDBC driver for the selected db_type cannot be resolved") + @ApiResponse(responseCode = "422", description = "JDBC driver for the selected db_type cannot be resolved, or a bytes-scanned cap on an engine without a bytes estimate (BYTES_SCANNED_CAP_NOT_SUPPORTED)") ResponseEntity createDatasource( @Valid @RequestBody CreateDatasourceRequest request, Authentication authentication, @@ -176,7 +176,9 @@ ResponseEntity createDatasource( request.resultCacheEnabled(), request.resultCacheTtlSeconds(), request.privateKeyPassphrase(), - request.environment()); + request.environment(), + request.maxBytesScannedPerQuery(), + request.bytesCapMissingEstimate()); var created = datasourceAdminService.create(command); recordAudit(AuditAction.DATASOURCE_CREATED, AuditResourceType.DATASOURCE, created.id(), caller, auditContext, Map.of("name", created.name(), "db_type", created.dbType().name())); @@ -206,6 +208,7 @@ DatasourceResponse getDatasource(@PathVariable UUID id, Authentication authentic @ApiResponse(responseCode = "400", description = "Validation error") @ApiResponse(responseCode = "404", description = "Datasource not found") @ApiResponse(responseCode = "409", description = "Name conflict with another datasource") + @ApiResponse(responseCode = "422", description = "Bytes-scanned cap on an engine without a bytes estimate (BYTES_SCANNED_CAP_NOT_SUPPORTED)") DatasourceResponse updateDatasource(@PathVariable UUID id, @Valid @RequestBody UpdateDatasourceRequest request, Authentication authentication, @@ -237,7 +240,10 @@ DatasourceResponse updateDatasource(@PathVariable UUID id, request.resultCacheTtlSeconds(), request.privateKeyPassphrase(), request.environment(), - request.clearEnvironment()); + request.clearEnvironment(), + request.maxBytesScannedPerQuery(), + request.clearMaxBytesScannedPerQuery(), + request.bytesCapMissingEstimate()); var updated = datasourceAdminService.update(id, caller.organizationId(), command); recordAudit(AuditAction.DATASOURCE_UPDATED, AuditResourceType.DATASOURCE, id, caller, auditContext, Map.of("name", updated.name())); @@ -335,7 +341,8 @@ PermissionListResponse listPermissions(@PathVariable UUID id, Authentication aut @ApiResponse(responseCode = "409", description = "Permission already exists for this user") @ApiResponse(responseCode = "422", description = "Target user is not in the organization, or denied_columns / " - + "denied_shapes is not supported by the datasource engine") + + "denied_shapes / bytes_scanned_limit_override is not supported by the " + + "datasource engine") ResponseEntity grantPermission( @PathVariable UUID id, @Valid @RequestBody CreatePermissionRequest request, @@ -349,6 +356,7 @@ ResponseEntity grantPermission( request.canDdl(), request.canBreakGlass(), request.rowLimitOverride(), + request.bytesScannedLimitOverride(), request.allowedSchemas(), request.allowedTables(), request.restrictedColumns(), @@ -425,8 +433,8 @@ GroupPermissionListResponse listGroupPermissions(@PathVariable UUID id, @ApiResponse(responseCode = "404", description = "Datasource or group not found") @ApiResponse(responseCode = "409", description = "Permission already exists for this group") @ApiResponse(responseCode = "422", - description = "denied_columns or denied_shapes is not supported by the datasource " - + "engine") + description = "denied_columns, denied_shapes or bytes_scanned_limit_override is not " + + "supported by the datasource engine") ResponseEntity grantGroupPermission( @PathVariable UUID id, @Valid @RequestBody CreateGroupPermissionRequest request, @@ -440,6 +448,7 @@ ResponseEntity grantGroupPermission( request.canDdl(), request.canBreakGlass(), request.rowLimitOverride(), + request.bytesScannedLimitOverride(), request.allowedSchemas(), request.allowedTables(), request.restrictedColumns(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java index 7dfbd79ba..668b5ca86 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandler.java @@ -20,6 +20,7 @@ import com.bablsoft.accessflow.core.api.EmailAlreadyExistsException; import com.bablsoft.accessflow.core.api.DeniedColumnsNotSupportedException; import com.bablsoft.accessflow.core.api.DeniedShapesNotSupportedException; +import com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException; import com.bablsoft.accessflow.core.api.IllegalDatasourcePermissionException; import com.bablsoft.accessflow.core.api.DataClassificationTagNotFoundException; import com.bablsoft.accessflow.core.api.IllegalDataClassificationTagException; @@ -433,6 +434,16 @@ ProblemDetail handleDeniedShapesNotSupported(DeniedShapesNotSupportedException e return pd; } + @ExceptionHandler(BytesScannedCapNotSupportedException.class) + ProblemDetail handleBytesScannedCapNotSupported(BytesScannedCapNotSupportedException ex) { + var pd = ProblemDetail.forStatusAndDetail(HttpStatus.UNPROCESSABLE_CONTENT, + msg("error.bytes_scanned_cap_not_supported", ex.dbType().name())); + pd.setProperty("error", "BYTES_SCANNED_CAP_NOT_SUPPORTED"); + pd.setProperty("dbType", ex.dbType().name()); + pd.setProperty("timestamp", Instant.now().toString()); + return pd; + } + @ExceptionHandler(MaskingPolicyNotFoundException.class) ProblemDetail handleMaskingPolicyNotFound(MaskingPolicyNotFoundException ex) { var pd = ProblemDetail.forStatusAndDetail(HttpStatus.NOT_FOUND, diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java index 3252e31d7..da157ef06 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateDatasourceRequest.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.security.internal.web.model; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.SslMode; @@ -59,5 +60,8 @@ public record CreateDatasourceRequest( @Size(max = 1024, message = "{validation.private_key_passphrase.max}") String privateKeyPassphrase, // #861: optional; an unset environment resolves to the org-wide default SQL review ruleset. - DatasourceEnvironment environment + DatasourceEnvironment environment, + // #941: optional bytes-scanned cap; only accepted on BigQuery / Snowflake / Databricks. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) {} diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java index 1db6b7b7c..2ed41091a 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreateGroupPermissionRequest.java @@ -19,6 +19,8 @@ public record CreateGroupPermissionRequest( Boolean canDdl, Boolean canBreakGlass, @Min(value = 1, message = "{validation.row_limit.min}") Integer rowLimitOverride, + // #941: bytes-scanned cap for this grant; only accepted on bytes-reporting engines. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List<@NotBlank(message = "{validation.restricted_columns.item_blank}") String> restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java index a5fa7e22f..66f66cd0e 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/CreatePermissionRequest.java @@ -19,6 +19,8 @@ public record CreatePermissionRequest( Boolean canDdl, Boolean canBreakGlass, @Min(value = 1, message = "{validation.row_limit.min}") Integer rowLimitOverride, + // #941: bytes-scanned cap for this grant; only accepted on bytes-reporting engines. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List<@NotBlank(message = "{validation.restricted_columns.item_blank}") String> restrictedColumns, diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java index d75db4538..27e409269 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/DatasourceResponse.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.security.internal.web.model; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.DatasourceView; import com.bablsoft.accessflow.core.api.DbType; @@ -36,7 +37,9 @@ public record DatasourceResponse( String localDatacenter, boolean resultCacheEnabled, Integer resultCacheTtlSeconds, - DatasourceEnvironment environment + DatasourceEnvironment environment, + Long maxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) { /** One read-replica endpoint — never carries the password. */ public record ReadReplicaResponse(UUID id, String jdbcUrl, String username) { @@ -73,6 +76,8 @@ public static DatasourceResponse from(DatasourceView view) { view.localDatacenter(), view.resultCacheEnabled(), view.resultCacheTtlSeconds(), - view.environment()); + view.environment(), + view.maxBytesScannedPerQuery(), + view.bytesCapMissingEstimate()); } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java index 75f6afd43..21e75d73b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/GroupPermissionResponse.java @@ -18,6 +18,7 @@ public record GroupPermissionResponse( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, @@ -41,6 +42,7 @@ public static GroupPermissionResponse from(DatasourceGroupPermissionView view) { view.canDdl(), view.canBreakGlass(), view.rowLimitOverride(), + view.bytesScannedLimitOverride(), view.allowedSchemas(), view.allowedTables(), view.restrictedColumns(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java index 4b4e2a939..430b82132 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/PermissionResponse.java @@ -18,6 +18,7 @@ public record PermissionResponse( boolean canDdl, boolean canBreakGlass, Integer rowLimitOverride, + Long bytesScannedLimitOverride, List allowedSchemas, List allowedTables, List restrictedColumns, @@ -41,6 +42,7 @@ public static PermissionResponse from(DatasourcePermissionView view) { view.canDdl(), view.canBreakGlass(), view.rowLimitOverride(), + view.bytesScannedLimitOverride(), view.allowedSchemas(), view.allowedTables(), view.restrictedColumns(), diff --git a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java index 00a4b2b07..5cfe9a335 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java +++ b/backend/src/main/java/com/bablsoft/accessflow/security/internal/web/model/UpdateDatasourceRequest.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.security.internal.web.model; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; import com.bablsoft.accessflow.core.api.DatasourceEnvironment; import com.bablsoft.accessflow.core.api.SslMode; import jakarta.validation.Valid; @@ -52,5 +53,9 @@ public record UpdateDatasourceRequest( String privateKeyPassphrase, // #861: null leaves the environment unchanged; clearEnvironment=true unsets it. DatasourceEnvironment environment, - Boolean clearEnvironment + Boolean clearEnvironment, + // #941: null leaves the cap unchanged; clearMaxBytesScannedPerQuery=true removes it. + @Min(value = 1, message = "{validation.bytes_cap.min}") Long maxBytesScannedPerQuery, + Boolean clearMaxBytesScannedPerQuery, + BytesCapMissingEstimateAction bytesCapMissingEstimate ) {} diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java index 1dea44e52..a7f31d68a 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionContext.java @@ -36,6 +36,10 @@ * {@code scanType} is the plan's root operation (e.g. {@code Seq Scan}). Both are {@code null} * when the estimate is absent, unsupported, or failed — the matching conditions fail closed. * + *

{@code estimatedBytesScanned} (#941) is the warehouse engines' pre-flight scan estimate in raw + * bytes from the same row — {@code null} for every engine that reports none, and whenever the + * estimate is absent or failed. + * *

{@code queryShapes} are the structural features re-derived from the SQL with the WHERE / LIMIT * signals (#940). {@code shapesAnalyzed} is {@code false} when the SQL could not be parsed or walked * (typically a non-SQL engine); the {@code query_shape} condition then fails closed. @@ -59,7 +63,8 @@ public record ConditionContext( Long estimatedRows, String scanType, Set queryShapes, - boolean shapesAnalyzed) { + boolean shapesAnalyzed, + Long estimatedBytesScanned) { public ConditionContext { referencedTables = Set.copyOf(referencedTables == null ? Set.of() : referencedTables); @@ -67,6 +72,21 @@ public record ConditionContext( queryShapes = Set.copyOf(queryShapes == null ? Set.of() : queryShapes); } + /** Backward-compatible constructor without the #941 bytes estimate (defaults to absent). */ + public ConditionContext(QueryType queryType, Set referencedTables, RiskLevel riskLevel, + int riskScore, String requesterRoleName, Set requesterGroupIds, + LocalDateTime evaluatedAt, boolean hasWhereClause, + boolean hasLimitClause, boolean transactional, + String requesterIpAddress, String requesterUserAgent, + boolean ciCdOrigin, Integer minutesSinceLastApproval, + boolean anomalyActive, Long estimatedRows, String scanType, + Set queryShapes, boolean shapesAnalyzed) { + this(queryType, referencedTables, riskLevel, riskScore, requesterRoleName, + requesterGroupIds, evaluatedAt, hasWhereClause, hasLimitClause, transactional, + requesterIpAddress, requesterUserAgent, ciCdOrigin, minutesSinceLastApproval, + anomalyActive, estimatedRows, scanType, queryShapes, shapesAnalyzed, null); + } + /** Backward-compatible constructor without the #940 shape signals (defaults to not analyzed). */ public ConditionContext(QueryType queryType, Set referencedTables, RiskLevel riskLevel, int riskScore, String requesterRoleName, Set requesterGroupIds, @@ -92,7 +112,7 @@ public ConditionContext(QueryType queryType, Set referencedTables, RiskL this(queryType, referencedTables, riskLevel, riskScore, requesterRoleName, requesterGroupIds, evaluatedAt, hasWhereClause, hasLimitClause, transactional, requesterIpAddress, requesterUserAgent, ciCdOrigin, minutesSinceLastApproval, - anomalyActive, null, null, Set.of(), false); + anomalyActive, null, null, Set.of(), false, null); } /** @return {@code true} when an AI risk level / score signal is present. */ diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java index a273ec809..1a9e3d3d2 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/ConditionNode.java @@ -215,6 +215,25 @@ record EstimatedRows(ComparisonOperator operator, long value) implements Conditi } } + /** + * Compares the warehouse engines' pre-flight bytes-scanned estimate (#941 — BigQuery dry-run + * {@code totalBytesProcessed}, Snowflake {@code bytesAssigned}, Databricks {@code sizeInBytes}) + * with {@code value}, in raw bytes. Fails closed: evaluates to {@code false} + * when no bytes estimate exists — every engine without one, a failed or unsupported estimate — + * so it is an escalation trigger, never a way to auto-approve on missing context. + */ + record EstimatedBytesScanned(ComparisonOperator operator, long value) implements ConditionNode { + public EstimatedBytesScanned { + if (operator == null) { + throw new IllegalArgumentException( + "EstimatedBytesScanned condition requires an operator"); + } + if (value < 0) { + throw new IllegalArgumentException("EstimatedBytesScanned value must be >= 0"); + } + } + } + /** * Matches when the pre-flight plan's root scan/operation type (AF-624 — e.g. {@code Seq Scan}, * {@code Index Scan}, {@code COLLSCAN}) matches any glob in {@code patterns} ({@code *} = any diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java index 71d336b9f..2b76abb70 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/api/QueryDecisionStepKind.java @@ -12,7 +12,8 @@ * missing stage is always a bug rather than a normal outcome. * *

Only {@link #ROUTING_POLICIES}, {@link #GRANT_FAST_PATH} and {@link #REVIEW_PLAN} are decided - * on the live asynchronous path; {@link #SQL_REVIEW} is evaluated synchronously at submission and + * on the live asynchronous path, with {@link #BYTES_SCANNED_CAP} decided just before them; + * {@link #SQL_REVIEW} is evaluated synchronously at submission and * read back there. The rest happen elsewhere in production — the first four in the * synchronous submission gate, {@link #ROW_SECURITY} and {@link #MASKING} at execution time, * {@link #ELIGIBLE_REVIEWERS} in notification fan-out, and {@link #BREAK_GLASS} in a separate @@ -40,6 +41,14 @@ public enum QueryDecisionStepKind implements DecisionStepKind { */ SQL_REVIEW, + /** + * The bytes-scanned cap (#941): {@code DENY} when the pre-flight estimate exceeds the cap, or + * no estimate exists and the datasource rejects on a missing one; {@code MATCH} when a missing + * estimate forces human review (every auto-approve stage below is then suppressed); + * {@code ALLOW} when the estimate is within the cap; {@code NO_MATCH} when no cap applies. + */ + BYTES_SCANNED_CAP, + /** Every enabled routing policy in ascending priority order, matched and unmatched. */ ROUTING_POLICIES, diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java new file mode 100644 index 000000000..731f35355 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheck.java @@ -0,0 +1,42 @@ +package com.bablsoft.accessflow.workflow.internal; + +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; + +import java.util.Objects; + +/** + * The bytes-scanned cap (#941) that applies to one request, and how its estimate compared — the + * input {@link QueryDecisionEvaluator} decides on, like the {@code BLOCK} rule ids. + * + * @param estimatedBytes the persisted pre-flight estimate, {@code null} when there is none + * @param outcome the comparison, or {@code null} when it cannot be made at all: the access + * simulator has no submitted query and therefore no estimate, so it reports + * the cap without letting a guessed outcome decide + */ +record BytesCapCheck(long limit, BytesScannedCapSource source, Long estimatedBytes, + BytesScannedCapOutcome outcome) { + + BytesCapCheck { + Objects.requireNonNull(source, "source"); + } + + static BytesCapCheck of(AppliedBytesCap cap, Long estimatedBytes) { + return new BytesCapCheck(cap.limit(), cap.source(), estimatedBytes, + cap.check(estimatedBytes)); + } + + /** A cap the caller knows applies but cannot compare (the simulator). */ + static BytesCapCheck unevaluated(AppliedBytesCap cap) { + return new BytesCapCheck(cap.limit(), cap.source(), null, null); + } + + boolean rejects() { + return outcome != null && outcome.rejects(); + } + + boolean forcesReview() { + return outcome == BytesScannedCapOutcome.NO_ESTIMATE_REVIEW; + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java index f8dec205d..532972844 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationService.java @@ -1,5 +1,6 @@ package com.bablsoft.accessflow.workflow.internal; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; import com.bablsoft.accessflow.core.api.ApproverRule; import com.bablsoft.accessflow.core.api.ColumnRefKeys; import com.bablsoft.accessflow.core.api.DatasourceAdminService; @@ -87,6 +88,7 @@ class DefaultAccessSimulationService implements AccessSimulationService { private final RowSecurityClassificationService rowSecurityClassificationService; private final MaskingPolicyResolutionService maskingPolicyResolutionService; private final BreakGlassEligibilityService breakGlassEligibilityService; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; // Time-of-day / day-of-week routing conditions evaluate in the server's local zone, so the // simulator has to use the same zone the live listener does. Deliberately NOT the injected @@ -140,9 +142,16 @@ public AccessSimulationResult simulate(UUID organizationId, AccessSimulationInpu .distinct() .sorted() .toList(); + // A hypothetical request has no pre-flight estimate (COST_ESTIMATE_ABSENT), so a bytes cap + // is reported but not compared: guessing "no estimate" would show a refusal or forced + // review that a real submission with an estimate would not get (#941). + var bytesCap = bytesScannedCapResolutionService + .resolve(input.datasourceId(), input.userId()) + .map(BytesCapCheck::unevaluated) + .orElse(null); var decision = queryDecisionEvaluator.evaluate( syntheticSnapshot(organizationId, input, parsed), input.aiOutcome(), - input.riskLevel(), input.effectiveRiskScore(), blockingRuleIds, clock); + input.riskLevel(), input.effectiveRiskScore(), blockingRuleIds, bytesCap, clock); steps.addAll(withFullPolicyList(decision, organizationId, input.datasourceId())); steps.add(reviewerStep(input, decision.nextStatus())); diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java index c47171bc5..1449a6bba 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleService.java @@ -1,5 +1,10 @@ package com.bablsoft.accessflow.workflow.internal; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; +import com.bablsoft.accessflow.core.api.BytesScannedCapExceededException; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; +import com.bablsoft.accessflow.core.api.QueryEstimateSnapshot; +import com.bablsoft.accessflow.proxy.api.QueryCostEstimateService; import com.bablsoft.accessflow.audit.api.AuditAction; import com.bablsoft.accessflow.audit.api.AuditEntry; import com.bablsoft.accessflow.audit.api.AuditLogService; @@ -87,6 +92,8 @@ class DefaultQueryLifecycleService implements QueryLifecycleService { private final ObjectMapper objectMapper; private final MessageSource messageSource; private final ApplicationEventPublisher eventPublisher; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; + private final QueryCostEstimateService queryCostEstimateService; private String msg(String key) { return messageSource.getMessage(key, null, LocaleContextHolder.getLocale()); @@ -277,6 +284,10 @@ private ExecutionOutcome doExecute(QueryRequestSnapshot query, UUID actorUserId, AuditAction successAction) { var startedAt = Instant.now(); try { + // #941: re-checked here, not only when the query left PENDING_AI — scheduled, recurring + // and break-glass runs execute later or without that decision, and the cap may have + // been lowered since. A refusal is recorded as a failed execution. + var bytesCap = enforceBytesScannedCap(query); var permission = permissionLookupService .findFor(query.submittedByUserId(), query.datasourceId()); var restrictedColumns = permission @@ -398,6 +409,13 @@ private ExecutionOutcome doExecute(QueryRequestSnapshot query, UUID actorUserId, appliedRowSecurityPolicyIds.stream() .map(UUID::toString).sorted().toList()); } + if (bytesCap != null) { + successMetadata.put("bytes_scanned_cap", bytesCap.limit()); + successMetadata.put("bytes_scanned_cap_source", bytesCap.source().name()); + if (bytesCap.estimatedBytes() != null) { + successMetadata.put("bytes_scanned_estimate", bytesCap.estimatedBytes()); + } + } if (!appliedRowLimitPolicyIds.isEmpty()) { successMetadata.put("applied_row_limit_policy_ids", appliedRowLimitPolicyIds.stream() @@ -423,6 +441,56 @@ private ExecutionOutcome doExecute(QueryRequestSnapshot query, UUID actorUserId, } } + /** + * @return the cap that applied and the estimate it was compared with, or {@code null} when no + * cap binds the submitter here + * @throws BytesScannedCapExceededException when the estimate exceeds the cap, or is missing and + * the datasource rejects on a missing estimate. A missing estimate under + * {@code REQUIRE_REVIEW} passes: the review it demands happened before approval. + */ + private BytesCapCheck enforceBytesScannedCap(QueryRequestSnapshot query) { + var cap = bytesScannedCapResolutionService + .resolve(query.datasourceId(), query.submittedByUserId()) + .orElse(null); + if (cap == null) { + return null; + } + Long estimated; + try { + estimated = queryCostEstimateService.estimateSubmittedQuery(query.id()) + .filter(e -> !e.failed()) + .map(QueryEstimateSnapshot::estimatedBytesScanned) + .orElse(null); + } catch (RuntimeException ex) { + log.warn("Pre-flight estimate unavailable for query {} at execution: {}", query.id(), + ex.getMessage()); + estimated = null; + } + var check = BytesCapCheck.of(cap, estimated); + if (!check.rejects()) { + return check; + } + var limit = ByteSizeFormat.format(cap.limit()); + var message = estimated == null + ? messageSource.getMessage("error.bytes_cap.no_estimate", new Object[]{limit}, + LocaleContextHolder.getLocale()) + : messageSource.getMessage("error.bytes_cap.exceeded", + new Object[]{ByteSizeFormat.format(estimated), limit}, + LocaleContextHolder.getLocale()); + var metadata = new HashMap(); + metadata.put("trigger", "bytes_scanned_cap"); + metadata.put("stage", "execution"); + metadata.put("limit", cap.limit()); + metadata.put("source", cap.source().name()); + if (estimated != null) { + metadata.put("estimated_bytes", estimated); + } + metadata.put("outcome", check.outcome().name()); + recordAudit(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, query.id(), null, + query.organizationId(), metadata); + throw new BytesScannedCapExceededException(message, cap, estimated, check.outcome()); + } + private ExecutionOutcome recordFailure(QueryRequestSnapshot query, UUID actorUserId, String trigger, Instant startedAt, RuntimeException ex) { var completedAt = Instant.now(); diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java index 483f1e0bd..afbd0d197 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecision.java @@ -25,17 +25,30 @@ * into human review (#864), or {@code null} when the finding changed * nothing — no block, {@code WARN} only, or the request was headed to * review regardless + * @param bytesCap the bytes-scanned cap that applied (#941), or {@code null} when none + * @param bytesCapChangedOutcome whether the cap refused the query or turned an automatic approval + * into human review — the condition for its audit row */ record QueryDecision(QueryDecisionKind kind, QueryStatus nextStatus, RoutingMatch routingMatch, Integer effectiveApprovals, UUID grantId, String grantApproverEmail, ConditionContext context, DecisionTrace trace, - SqlReviewSuppression sqlReviewSuppression) { + SqlReviewSuppression sqlReviewSuppression, BytesCapCheck bytesCap, + boolean bytesCapChangedOutcome) { + + /** A decision no bytes-scanned cap took part in. */ + QueryDecision(QueryDecisionKind kind, QueryStatus nextStatus, RoutingMatch routingMatch, + Integer effectiveApprovals, UUID grantId, String grantApproverEmail, + ConditionContext context, DecisionTrace trace, + SqlReviewSuppression sqlReviewSuppression) { + this(kind, nextStatus, routingMatch, effectiveApprovals, grantId, grantApproverEmail, context, + trace, sqlReviewSuppression, null, false); + } /** A decision no SQL review finding interfered with. */ QueryDecision(QueryDecisionKind kind, QueryStatus nextStatus, RoutingMatch routingMatch, Integer effectiveApprovals, UUID grantId, String grantApproverEmail, ConditionContext context, DecisionTrace trace) { this(kind, nextStatus, routingMatch, effectiveApprovals, grantId, grantApproverEmail, context, - trace, null); + trace, (SqlReviewSuppression) null); } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java index 357b66216..bdfd66c54 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluator.java @@ -2,6 +2,7 @@ import com.bablsoft.accessflow.access.api.AccessGrantLookupService; import com.bablsoft.accessflow.access.api.AccessGrantView; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; import com.bablsoft.accessflow.core.api.QueryRequestSnapshot; import com.bablsoft.accessflow.core.api.QueryStatus; import com.bablsoft.accessflow.core.api.QueryType; @@ -68,17 +69,31 @@ class QueryDecisionEvaluator { private final RoutingPolicyEngine routingPolicyEngine; private final AccessGrantLookupService accessGrantLookupService; + /** {@link #evaluate(QueryRequestSnapshot, AiOutcome, RiskLevel, int, List, BytesCapCheck, Clock)} with no bytes cap. */ + QueryDecision evaluate(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLevel riskLevel, + int riskScore, List blockingRuleIds, Clock clock) { + return evaluate(query, aiOutcome, riskLevel, riskScore, blockingRuleIds, null, clock); + } + /** * @param riskScore the AI's numeric score, or {@code -1} when there is none — the same * "absent" sentinel the live completion event uses * @param blockingRuleIds the distinct SQL review rule ids that fired at {@code BLOCK} for this * request, empty when none did (#864) + * @param bytesCap the bytes-scanned cap that applies (#941), {@code null} when none does */ QueryDecision evaluate(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLevel riskLevel, - int riskScore, List blockingRuleIds, Clock clock) { + int riskScore, List blockingRuleIds, BytesCapCheck bytesCap, + Clock clock) { var block = blockingRuleIds == null ? List.of() : List.copyOf(blockingRuleIds); + var guard = new Guard(block, bytesCap); + // A hard cap is a refusal, not a routing signal: it decides before routing and before the + // AI-failure fallback, so no policy or plan can approve a query the cap has refused. + if (bytesCap != null && bytesCap.rejects()) { + return bytesCapRejected(block, bytesCap); + } if (aiOutcome == AiOutcome.FAILED) { - return aiFailed(block); + return aiFailed(block, bytesCap); } // Only a COMPLETED analysis carries a risk signal. Normalising here rather than trusting the // caller keeps the SKIPPED branch identical to production, where the listener passes no risk @@ -88,28 +103,100 @@ QueryDecision evaluate(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLeve var plan = reviewPlanLookupService.findForDatasource(query.datasourceId()).orElse(null); var context = conditionContextFactory.forLiveQuery(query, effectiveRisk, effectiveScore, clock); - var steps = new ArrayList(4); + var steps = new ArrayList(5); steps.add(sqlReviewStep(block)); + steps.add(bytesCapStep(bytesCap)); var match = routingPolicyEngine.evaluate(query.organizationId(), query.datasourceId(), context).orElse(null); if (match != null) { - return routed(match, plan, context, steps, block); + return routed(match, plan, context, steps, guard); } steps.add(DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.NO_MATCH, "workflow.decision.routing.no_match")); var suppressed = new ArrayList(2); - var grant = findCoveringGrant(query, context, steps, block, suppressed); + var grant = findCoveringGrant(query, context, steps, guard, suppressed); if (grant != null) { steps.add(DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.SKIP, "workflow.decision.plan.skipped_grant_covered", planDetails(plan))); return new QueryDecision(QueryDecisionKind.GRANT_FAST_PATH, QueryStatus.APPROVED, null, null, grant.id(), grant.approverEmail(), context, - new DecisionTrace(steps, QueryStatus.APPROVED)); + new DecisionTrace(steps, QueryStatus.APPROVED), null, bytesCap, false); + } + + return planned(query, plan, effectiveRisk, context, steps, guard, suppressed); + } + + /** + * What may turn an automatic approval into human review: a {@code BLOCK} SQL review finding + * (#864) and a bytes-scanned cap with no estimate to compare against (#941). The trace names + * the SQL review first when both apply. + */ + private record Guard(List block, BytesCapCheck bytesCap) { + + boolean suppresses() { + return !block.isEmpty() || (bytesCap != null && bytesCap.forcesReview()); + } + + boolean capForcesReview() { + return bytesCap != null && bytesCap.forcesReview(); } - return planned(query, plan, effectiveRisk, context, steps, block, suppressed); + String reasonSuffix() { + return block.isEmpty() ? "bytes_cap" : "sql_review"; + } + } + + /** The cap step is always present so the trace keeps one entry per stage. */ + private static DecisionTraceStep bytesCapStep(BytesCapCheck cap) { + if (cap == null) { + return DecisionTraceStep.of(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.NO_MATCH, "workflow.decision.bytes_cap.none"); + } + var details = new LinkedHashMap(); + details.put("bytes_scanned_cap", cap.limit()); + details.put("bytes_scanned_cap_source", cap.source().name()); + details.put("estimated_bytes_scanned", cap.estimatedBytes()); + details.put("bytes_scanned_cap_outcome", cap.outcome() == null ? null : cap.outcome().name()); + var limit = ByteSizeFormat.format(cap.limit()); + if (cap.outcome() == null) { + return new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.SKIP, + "workflow.decision.bytes_cap.unevaluated", List.of(limit), details); + } + return switch (cap.outcome()) { + case WITHIN -> new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.ALLOW, "workflow.decision.bytes_cap.within", + List.of(ByteSizeFormat.format(cap.estimatedBytes()), limit), details); + case EXCEEDED -> new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.DENY, "workflow.decision.bytes_cap.exceeded", + List.of(ByteSizeFormat.format(cap.estimatedBytes()), limit), details); + case NO_ESTIMATE_REVIEW -> new DecisionTraceStep(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.MATCH, "workflow.decision.bytes_cap.no_estimate_review", + List.of(limit), details); + case NO_ESTIMATE_REJECTED -> new DecisionTraceStep( + QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.DENY, + "workflow.decision.bytes_cap.no_estimate_rejected", List.of(limit), details); + }; + } + + /** + * The cap refused the query (#941). Nothing downstream runs — there is no routing decision to + * record and no plan to consult — which is why the context is not even built. + */ + private static QueryDecision bytesCapRejected(List block, BytesCapCheck cap) { + var steps = List.of( + sqlReviewStep(block), + bytesCapStep(cap), + DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.SKIP, + "workflow.decision.routing.skipped_bytes_cap"), + DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.SKIP, + "workflow.decision.grant.skipped_bytes_cap"), + DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.SKIP, + "workflow.decision.plan.skipped_bytes_cap")); + return new QueryDecision(QueryDecisionKind.BYTES_CAP_REJECTED, QueryStatus.REJECTED, null, + null, null, null, null, new DecisionTrace(steps, QueryStatus.REJECTED), null, cap, + true); } /** @@ -131,7 +218,7 @@ private static DecisionTraceStep sqlReviewStep(List block) { private static SqlReviewSuppression suppression(List block, List paths) { - return paths.isEmpty() ? null : new SqlReviewSuppression(block, paths); + return paths.isEmpty() || block.isEmpty() ? null : new SqlReviewSuppression(block, paths); } /** @@ -140,9 +227,10 @@ private static SqlReviewSuppression suppression(List block, * auto-decision signal — and neither does the grant fast path or the review plan. Decided before * any lookup, mirroring the live listener, which builds no context at all. */ - private static QueryDecision aiFailed(List block) { + private static QueryDecision aiFailed(List block, BytesCapCheck bytesCap) { var steps = List.of( sqlReviewStep(block), + bytesCapStep(bytesCap), DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.SKIP, "workflow.decision.routing.skipped_ai_failed"), DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.SKIP, @@ -151,7 +239,7 @@ private static QueryDecision aiFailed(List block) { "workflow.decision.plan.skipped_ai_failed")); return new QueryDecision(QueryDecisionKind.AI_FAILED_PENDING_REVIEW, QueryStatus.PENDING_REVIEW, null, null, null, null, null, - new DecisionTrace(steps, QueryStatus.PENDING_REVIEW)); + new DecisionTrace(steps, QueryStatus.PENDING_REVIEW), null, bytesCap, false); } /** @@ -161,8 +249,10 @@ private static QueryDecision aiFailed(List block) { */ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, ConditionContext context, List steps, - List block) { - boolean suppressedApprove = match.action() == RoutingAction.AUTO_APPROVE && !block.isEmpty(); + Guard guard) { + var block = guard.block(); + boolean suppressedApprove = match.action() == RoutingAction.AUTO_APPROVE + && guard.suppresses(); var effect = switch (match.action()) { case AUTO_APPROVE -> suppressedApprove ? new RoutedEffect(QueryDecisionKind.ROUTING_AUTO_APPROVE_SUPPRESSED, @@ -184,10 +274,13 @@ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, details.put("matched_policy_name", match.policyName()); details.put("action", match.action().name()); details.put("effective_min_approvals", effective); - details.put("sql_review_suppressed", suppressedApprove); + details.put("sql_review_suppressed", suppressedApprove && !block.isEmpty()); + details.put("bytes_cap_suppressed", suppressedApprove && guard.capForcesReview()); steps.add(suppressedApprove ? new DecisionTraceStep(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.MATCH, - "workflow.decision.routing.matched_auto_approve_suppressed", + block.isEmpty() + ? "workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap" + : "workflow.decision.routing.matched_auto_approve_suppressed", List.of(String.valueOf(match.policyName())), details) : new DecisionTraceStep(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.MATCH, "workflow.decision.routing.matched", @@ -198,9 +291,10 @@ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, "workflow.decision.plan.skipped_routing_decided", planDetails(plan))); return new QueryDecision(kind, nextStatus, match, effective, null, null, context, new DecisionTrace(steps, nextStatus), - suppressedApprove + suppressedApprove && !block.isEmpty() ? new SqlReviewSuppression(block, List.of(SuppressedAutoApproval.ROUTING_AUTO_APPROVE)) - : null); + : null, + guard.bytesCap(), suppressedApprove && guard.capForcesReview()); } /** @@ -217,7 +311,7 @@ private QueryDecision routed(RoutingMatch match, ReviewPlanSnapshot plan, * @return the first covering grant, or {@code null} to fall through to the review plan */ private AccessGrantView findCoveringGrant(QueryRequestSnapshot query, ConditionContext context, - List steps, List block, + List steps, Guard guard, List suppressed) { if (context.anomalyActive()) { steps.add(DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.NO_MATCH, @@ -256,10 +350,11 @@ private AccessGrantView findCoveringGrant(QueryRequestSnapshot query, ConditionC var details = consideredGrants(grants); details.put("grant_id", grant.id()); details.put("approver_email", grant.approverEmail()); - if (!block.isEmpty()) { + if (guard.suppresses()) { suppressed.add(SuppressedAutoApproval.GRANT_FAST_PATH); steps.add(new DecisionTraceStep(QueryDecisionStepKind.GRANT_FAST_PATH, - StepOutcome.NO_MATCH, "workflow.decision.grant.suppressed_sql_review", + StepOutcome.NO_MATCH, + "workflow.decision.grant.suppressed_" + guard.reasonSuffix(), List.of(String.valueOf(grant.id())), details)); return null; } @@ -281,8 +376,9 @@ private record RoutedEffect(QueryDecisionKind kind, QueryStatus nextStatus, private QueryDecision planned(QueryRequestSnapshot query, ReviewPlanSnapshot plan, RiskLevel riskLevel, ConditionContext context, - List steps, List block, + List steps, Guard guard, List suppressed) { + var block = guard.block(); var details = planDetails(plan); QueryStatus nextStatus; String reasonKey; @@ -302,12 +398,14 @@ private QueryDecision planned(QueryRequestSnapshot query, ReviewPlanSnapshot pla } // Decided un-guarded first so the trace says which plan rule WOULD have approved; the block // then overrides the status alone (#864). - if (nextStatus == QueryStatus.APPROVED && !block.isEmpty()) { + if (nextStatus == QueryStatus.APPROVED && guard.suppresses()) { suppressed.add(SuppressedAutoApproval.REVIEW_PLAN); nextStatus = QueryStatus.PENDING_REVIEW; - reasonKey = "workflow.decision.plan.suppressed_sql_review"; + reasonKey = "workflow.decision.plan.suppressed_" + guard.reasonSuffix(); } - details.put("sql_review_suppressed", suppressed.contains(SuppressedAutoApproval.REVIEW_PLAN)); + boolean planSuppressed = suppressed.contains(SuppressedAutoApproval.REVIEW_PLAN); + details.put("sql_review_suppressed", planSuppressed && !block.isEmpty()); + details.put("bytes_cap_suppressed", planSuppressed && guard.capForcesReview()); steps.add(DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, nextStatus == QueryStatus.APPROVED ? StepOutcome.ALLOW : StepOutcome.DENY, reasonKey, details)); @@ -315,7 +413,8 @@ private QueryDecision planned(QueryRequestSnapshot query, ReviewPlanSnapshot pla ? QueryDecisionKind.PLAN_APPROVED : QueryDecisionKind.PLAN_PENDING_REVIEW; return new QueryDecision(kind, nextStatus, null, null, null, null, context, - new DecisionTrace(steps, nextStatus), suppression(block, suppressed)); + new DecisionTrace(steps, nextStatus), suppression(block, suppressed), + guard.bytesCap(), guard.capForcesReview() && !suppressed.isEmpty()); } private static LinkedHashMap consideredGrants(List grants) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java index 8d56b11bb..e79fd50d1 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionKind.java @@ -30,5 +30,11 @@ enum QueryDecisionKind { PLAN_PENDING_REVIEW, /** AI analysis failed; the query goes to human review unconditionally. */ - AI_FAILED_PENDING_REVIEW + AI_FAILED_PENDING_REVIEW, + + /** + * The bytes-scanned cap (#941) refused the query: its estimate exceeds the cap, or it has none + * and the datasource rejects on a missing estimate. Decided before routing and AI failure. + */ + BYTES_CAP_REJECTED } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java index f908bbdf8..76d68ce68 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachine.java @@ -4,7 +4,14 @@ import com.bablsoft.accessflow.audit.api.AuditEntry; import com.bablsoft.accessflow.audit.api.AuditLogService; import com.bablsoft.accessflow.audit.api.AuditResourceType; +import com.bablsoft.accessflow.core.api.ByteSizeFormat; +import com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService; +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.QueryEstimateLookupService; +import com.bablsoft.accessflow.core.api.QueryEstimateSnapshot; import com.bablsoft.accessflow.core.api.QueryRequestLookupService; +import com.bablsoft.accessflow.core.api.RiskLevel; +import com.bablsoft.accessflow.proxy.api.QueryCostEstimateService; import com.bablsoft.accessflow.core.api.QueryRequestSnapshot; import com.bablsoft.accessflow.core.api.QueryRequestStateService; import com.bablsoft.accessflow.core.api.QueryStatus; @@ -25,6 +32,9 @@ import org.springframework.context.MessageSource; import org.springframework.modulith.events.ApplicationModuleListener; import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.time.Clock; import java.util.LinkedHashMap; @@ -53,6 +63,11 @@ * when any are present. When that actually changed the outcome, one {@code SQL_REVIEW_BLOCKED} audit * row is written here — system-attributed, {@code trigger=sql_review}. * + *

The bytes-scanned cap (#941) is resolved and compared against the persisted pre-flight estimate + * at every entry point and handed to the evaluator; the cap, its source and the comparison are + * stamped on the query before the decision is applied, and a {@code QUERY_BYTES_SCANNED_CAP_ENFORCED} + * audit row is written when the cap refused the query or forced it to review. + * *

AI failure unconditionally lands in {@code PENDING_REVIEW} so a human can inspect the query. The * skipped path (datasource has {@code ai_analysis_enabled = false}) runs routing with no risk signal * — risk-based conditions evaluate to {@code false} — and otherwise respects @@ -73,6 +88,10 @@ class QueryReviewStateMachine { private final AuditLogService auditLogService; private final MessageSource messageSource; private final ApplicationEventPublisher eventPublisher; + private final BytesScannedCapResolutionService bytesScannedCapResolutionService; + private final QueryCostEstimateService queryCostEstimateService; + private final QueryEstimateLookupService queryEstimateLookupService; + private final PlatformTransactionManager transactionManager; // Time-of-day / day-of-week routing conditions evaluate in the server's local zone. A field // (not an injected bean) so it can be overridden in tests without colliding with the proxy's @@ -85,29 +104,97 @@ void setClock(Clock clock) { @ApplicationModuleListener void onAiCompleted(AiAnalysisCompletedEvent event) { + // The AI analyzer computed the estimate before publishing, so it is only read here. + var cap = prepare(event.queryRequestId(), false); var query = load(event.queryRequestId(), "AiAnalysisCompletedEvent"); if (query != null) { - apply(query, queryDecisionEvaluator.evaluate(query, AiOutcome.COMPLETED, - event.riskLevel(), event.riskScore(), blockingRuleIds(query), clock)); + decide(query, AiOutcome.COMPLETED, event.riskLevel(), event.riskScore(), cap); } } @ApplicationModuleListener void onAiSkipped(AiAnalysisSkippedEvent event) { + // With AI off nothing has waited for the pre-flight estimate: an independent listener + // computes it, and routing would race it, so an estimated_rows / estimated_bytes_scanned + // policy would silently fail closed on an estimate that was about to exist (#941). + var cap = prepare(event.queryRequestId(), true); var query = load(event.queryRequestId(), "AiAnalysisSkippedEvent"); if (query != null) { - apply(query, queryDecisionEvaluator.evaluate(query, AiOutcome.SKIPPED, null, -1, - blockingRuleIds(query), clock)); + decide(query, AiOutcome.SKIPPED, null, -1, cap); } } @ApplicationModuleListener void onAiFailed(AiAnalysisFailedEvent event) { + var cap = prepare(event.queryRequestId(), false); var query = load(event.queryRequestId(), "AiAnalysisFailedEvent"); if (query != null) { - apply(query, queryDecisionEvaluator.evaluate(query, AiOutcome.FAILED, null, -1, - blockingRuleIds(query), clock)); + decide(query, AiOutcome.FAILED, null, -1, cap); + } + } + + /** + * Resolves the bytes-scanned cap (#941) and makes sure the pre-flight estimate exists — when AI + * was skipped, or when a cap applies — before the decision transaction reads anything. + * + *

It runs in its own transaction on purpose. The estimate write is a check-then-insert + * that races the independent estimate listener on a unique key, and it updates the + * version-checked {@code query_requests} row. Inside the decision transaction a lost race would + * mark that transaction rollback-only — and with it the transition, stranding the query in + * {@code PENDING_AI} — while a won race would leave the decision holding a stale query row. + * Here a lost race only fails this inner transaction; the winner's row is read afterwards. + */ + private AppliedBytesCap prepare(UUID queryRequestId, boolean alwaysEstimate) { + var template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + try { + return template.execute(status -> { + var query = queryRequestLookupService.findById(queryRequestId).orElse(null); + if (query == null || query.status() != QueryStatus.PENDING_AI) { + return null; + } + var cap = bytesScannedCapResolutionService + .resolve(query.datasourceId(), query.submittedByUserId()) + .orElse(null); + if (alwaysEstimate || cap != null) { + queryCostEstimateService.estimateSubmittedQuery(queryRequestId); + } + return cap; + }); + } catch (RuntimeException ex) { + // Typically the estimate listener won the insert race. The cap itself is re-resolved + // below; a missing estimate is a missing estimate, never an error. + log.warn("Pre-flight estimate preparation failed for query {}: {}", queryRequestId, + ex.getMessage()); + return resolveQuietly(queryRequestId); + } + } + + private AppliedBytesCap resolveQuietly(UUID queryRequestId) { + return queryRequestLookupService.findById(queryRequestId) + .flatMap(q -> bytesScannedCapResolutionService.resolve(q.datasourceId(), + q.submittedByUserId())) + .orElse(null); + } + + private void decide(QueryRequestSnapshot query, AiOutcome aiOutcome, RiskLevel riskLevel, + int riskScore, AppliedBytesCap cap) { + var bytesCap = cap == null ? null : BytesCapCheck.of(cap, estimatedBytes(query)); + var decision = queryDecisionEvaluator.evaluate(query, aiOutcome, riskLevel, riskScore, + blockingRuleIds(query), bytesCap, clock); + if (bytesCap != null) { + queryRequestStateService.recordBytesScannedCap(query.id(), bytesCap.limit(), + bytesCap.source(), bytesCap.outcome()); } + apply(query, decision); + } + + /** The persisted bytes estimate; absent, failed or unsupported all read as "none". */ + private Long estimatedBytes(QueryRequestSnapshot query) { + return queryEstimateLookupService.findByQueryRequestId(query.id()) + .filter(e -> !e.failed()) + .map(QueryEstimateSnapshot::estimatedBytesScanned) + .orElse(null); } private List blockingRuleIds(QueryRequestSnapshot query) { @@ -173,6 +260,12 @@ private void apply(QueryRequestSnapshot query, QueryDecision decision) { ? new QueryAutoApprovedEvent(query.id()) : new QueryReadyForReviewEvent(query.id())); } + case BYTES_CAP_REJECTED -> { + queryRequestStateService.transitionTo(query.id(), QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + eventPublisher.publishEvent(new QueryAutoRejectedEvent(query.id(), null, + bytesCapReason(decision.bytesCap()))); + } // A switch STATEMENT over an enum is not exhaustiveness-checked, so a new kind would // otherwise fall through silently and strand the query in PENDING_AI forever. default -> throw new IllegalStateException("Unhandled decision kind " + decision.kind()); @@ -180,6 +273,9 @@ private void apply(QueryRequestSnapshot query, QueryDecision decision) { if (decision.sqlReviewSuppression() != null) { auditSqlReviewBlocked(query, decision); } + if (decision.bytesCapChangedOutcome()) { + auditBytesCapEnforced(query, decision); + } // Logged after the fact: a line claiming a query was auto-approved must not outlive a // persistence call that then failed. if (match != null) { @@ -220,6 +316,46 @@ private void auditSqlReviewBlocked(QueryRequestSnapshot query, QueryDecision dec } } + /** + * One row per query whose outcome the bytes-scanned cap changed (#941) — a refusal, or an + * automatic approval turned into review. Swallow-and-log, like the SQL review row above. + */ + private void auditBytesCapEnforced(QueryRequestSnapshot query, QueryDecision decision) { + var cap = decision.bytesCap(); + var metadata = new LinkedHashMap(); + metadata.put("trigger", "bytes_scanned_cap"); + metadata.put("stage", "decision"); + metadata.put("limit", cap.limit()); + metadata.put("source", cap.source().name()); + if (cap.estimatedBytes() != null) { + metadata.put("estimated_bytes", cap.estimatedBytes()); + } + metadata.put("outcome", cap.outcome().name()); + if (decision.routingMatch() != null) { + metadata.put("matched_policy_id", decision.routingMatch().policyId()); + } + try { + auditLogService.record(new AuditEntry(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, + AuditResourceType.QUERY_REQUEST, query.id(), query.organizationId(), null, + metadata, null, null)); + } catch (RuntimeException ex) { + log.error("Audit write failed for QUERY_BYTES_SCANNED_CAP_ENFORCED on query {}", + query.id(), ex); + } + } + + /** Server-default locale for the same reason as {@link #grantReason}. */ + private String bytesCapReason(BytesCapCheck cap) { + var limit = ByteSizeFormat.format(cap.limit()); + if (cap.estimatedBytes() == null) { + return messageSource.getMessage("workflow.bytes_cap.rejected_no_estimate", + new Object[]{limit}, Locale.getDefault()); + } + return messageSource.getMessage("workflow.bytes_cap.rejected_exceeded", + new Object[]{ByteSizeFormat.format(cap.estimatedBytes()), limit}, + Locale.getDefault()); + } + /** * Rendered here rather than in the evaluator: there is no request locale on this asynchronous * path, so the reason falls back to the server default, and the evaluator stays free of diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java index 7b5d16544..f631a013c 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactory.java @@ -64,7 +64,8 @@ public ConditionContext forLiveQuery(QueryRequestSnapshot query, RiskLevel riskL query.datasourceId(), query.id(), clock.instant()), behaviorAnomalyLookupService.hasActiveAnomaly(query.organizationId(), query.submittedByUserId(), query.datasourceId()), - estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed()); + estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed(), + estimate.bytesScanned()); } /** @@ -87,13 +88,14 @@ public ConditionContext forHistoricalRow(QueryCorpusRow row, ZoneId zone) { parsed.transactional(), row.submittedIp(), row.submittedUserAgent(), row.ciCdOrigin(), minutesSinceLastApproval(row.organizationId(), row.submittedByUserId(), row.datasourceId(), row.id(), row.createdAt()), - false, estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed()); + false, estimate.rows(), estimate.scanType(), parsed.shapes(), parsed.shapesAnalyzed(), + estimate.bytesScanned()); } /** * AF-624 pre-flight estimate signals. The estimate pipeline runs independently of AI analysis, * so whatever is persisted for the query is the signal; absent / unsupported / failed rows - * leave both fields null and the matching conditions fail closed. + * leave every field null and the matching conditions fail closed. * *

The replay arm reads it too: unlike membership or the anomaly flag, the estimate is a * persisted per-query fact, so dropping it would make an {@code estimated_rows} policy simulate @@ -102,15 +104,15 @@ public ConditionContext forHistoricalRow(QueryCorpusRow row, ZoneId zone) { private EstimateSignals estimateSignals(UUID queryRequestId) { var estimate = queryEstimateLookupService.findByQueryRequestId(queryRequestId).orElse(null); if (estimate == null || estimate.failed()) { - return new EstimateSignals(null, null); + return new EstimateSignals(null, null, null); } var rows = estimate.affectedRowCount() != null ? estimate.affectedRowCount() : estimate.estimatedRows(); - return new EstimateSignals(rows, estimate.scanType()); + return new EstimateSignals(rows, estimate.scanType(), estimate.estimatedBytesScanned()); } - private record EstimateSignals(Long rows, String scanType) { + private record EstimateSignals(Long rows, String scanType, Long bytesScanned) { } private String roleName(UUID userId) { diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java index 1d5db9596..0befada1b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionNodeMixin.java @@ -34,6 +34,7 @@ @JsonSubTypes.Type(value = ConditionNode.CiCdOrigin.class, name = "cicd_origin"), @JsonSubTypes.Type(value = ConditionNode.AnomalyDetected.class, name = "anomaly_detected"), @JsonSubTypes.Type(value = ConditionNode.EstimatedRows.class, name = "estimated_rows"), + @JsonSubTypes.Type(value = ConditionNode.EstimatedBytesScanned.class, name = "estimated_bytes_scanned"), @JsonSubTypes.Type(value = ConditionNode.ScanTypeMatches.class, name = "scan_type") }) interface ConditionNodeMixin { diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java index 339402976..69eb09520 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluator.java @@ -53,6 +53,8 @@ public boolean matches(ConditionNode node, ConditionContext ctx) { case ConditionNode.AnomalyDetected c -> ctx.anomalyActive() == c.expected(); case ConditionNode.EstimatedRows c -> ctx.hasEstimateSignal() && c.operator().test(ctx.estimatedRows(), c.value()); + case ConditionNode.EstimatedBytesScanned c -> ctx.estimatedBytesScanned() != null + && c.operator().test(ctx.estimatedBytesScanned(), c.value()); case ConditionNode.ScanTypeMatches c -> matchesScanType(c, ctx); }; } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java index c41468144..2a8f69645 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/AccessSimulationResponse.java @@ -68,7 +68,8 @@ record EvaluatedContext(com.bablsoft.accessflow.core.api.QueryType queryType, String requesterIpAddress, String requesterUserAgent, boolean ciCdOrigin, Integer minutesSinceLastApproval, boolean anomalyActive, Long estimatedRows, String scanType, - List queryShapes, boolean shapesAnalyzed) { + List queryShapes, boolean shapesAnalyzed, + Long estimatedBytesScanned) { static EvaluatedContext from(ConditionContext context) { if (context == null) { @@ -82,7 +83,8 @@ static EvaluatedContext from(ConditionContext context) { context.requesterIpAddress(), context.requesterUserAgent(), context.ciCdOrigin(), context.minutesSinceLastApproval(), context.anomalyActive(), context.estimatedRows(), context.scanType(), - context.queryShapes().stream().sorted().toList(), context.shapesAnalyzed()); + context.queryShapes().stream().sorted().toList(), context.shapesAnalyzed(), + context.estimatedBytesScanned()); } } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java index 435ceb3d0..2ae42ef5c 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java +++ b/backend/src/main/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponse.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonRawValue; import com.bablsoft.accessflow.access.api.AccessGrantView; import com.bablsoft.accessflow.core.api.ApplicationNameSource; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.AiProviderType; import com.bablsoft.accessflow.core.api.DbType; import com.bablsoft.accessflow.core.api.DecisionType; @@ -63,7 +65,9 @@ public record QueryDetailResponse( /** The calling application (#938); null when unknown. */ String applicationName, /** {@code API_KEY} (trustworthy) or {@code HEADER} (client-controlled). */ - ApplicationNameSource applicationNameSource) { + ApplicationNameSource applicationNameSource, + /** The bytes-scanned cap (#941) that applied to this query; null when none did. */ + BytesScannedCapDetail bytesScannedCap) { public static QueryDetailResponse from(QueryDetailView view) { return from(view, null, null); @@ -161,7 +165,21 @@ public static QueryDetailResponse from(QueryDetailView view, MatchedRoutingPolic view.onBehalfOfUserId() == null ? null : new OnBehalfOfRef(view.onBehalfOfUserId(), view.onBehalfOfEmail()), view.applicationName(), - view.applicationNameSource()); + view.applicationNameSource(), + BytesScannedCapDetail.from(view.bytesScannedCap())); + } + + /** + * The bytes-scanned cap (#941) recorded when the query left {@code PENDING_AI}: the limit, the + * configuration it came from, and how the estimate compared. + */ + public record BytesScannedCapDetail(long limit, BytesScannedCapSource source, + BytesScannedCapOutcome outcome) { + + static BytesScannedCapDetail from(QueryDetailView.BytesScannedCapDetail src) { + return src == null ? null + : new BytesScannedCapDetail(src.limit(), src.source(), src.outcome()); + } } /** A ticket auto-created in an external ticketing system for this query (AF-453). */ @@ -285,7 +303,8 @@ public record CostEstimateDetail( String unsupportedReason, boolean failed, String errorMessage, - Integer durationMs) { + Integer durationMs, + Long estimatedBytesScanned) { static CostEstimateDetail from(QueryDetailView.CostEstimateDetail src) { if (src == null) { @@ -305,7 +324,8 @@ static CostEstimateDetail from(QueryDetailView.CostEstimateDetail src) { src.unsupportedReason(), src.failed(), src.errorMessage(), - src.durationMs()); + src.durationMs(), + src.estimatedBytesScanned()); } } diff --git a/backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql b/backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql new file mode 100644 index 000000000..d9d287fae --- /dev/null +++ b/backend/src/main/resources/db/migration/V192__add_bytes_scanned_caps.sql @@ -0,0 +1,25 @@ +-- Bytes-scanned cost caps (#941). The warehouse engines' pre-flight scan estimate (AF-634) is now +-- persisted with the query's estimate, and an admin may cap it per datasource and per grant +-- (most restrictive wins; NULL = no cap). The cap is only configurable on engines that report a +-- bytes estimate (BigQuery, Snowflake, Databricks) — enforced by the service, not here. +ALTER TABLE query_estimates ADD COLUMN estimated_bytes_scanned BIGINT; + +CREATE TYPE bytes_cap_missing_estimate_action AS ENUM ('REQUIRE_REVIEW', 'REJECT'); +CREATE TYPE bytes_scanned_cap_source AS ENUM ('DATASOURCE', 'GRANT'); +CREATE TYPE bytes_scanned_cap_outcome AS ENUM ('WITHIN', 'EXCEEDED', 'NO_ESTIMATE_REVIEW', + 'NO_ESTIMATE_REJECTED'); + +ALTER TABLE datasources ADD COLUMN max_bytes_scanned_per_query BIGINT + CONSTRAINT chk_datasources_max_bytes_scanned_positive CHECK (max_bytes_scanned_per_query > 0); +ALTER TABLE datasources ADD COLUMN bytes_cap_missing_estimate bytes_cap_missing_estimate_action + NOT NULL DEFAULT 'REQUIRE_REVIEW'; + +ALTER TABLE datasource_user_permissions ADD COLUMN bytes_scanned_limit_override BIGINT + CONSTRAINT chk_dup_bytes_scanned_limit_positive CHECK (bytes_scanned_limit_override > 0); +ALTER TABLE datasource_group_permissions ADD COLUMN bytes_scanned_limit_override BIGINT + CONSTRAINT chk_dgp_bytes_scanned_limit_positive CHECK (bytes_scanned_limit_override > 0); + +-- Stamped when the query leaves PENDING_AI and a cap applied, so the detail view can say why. +ALTER TABLE query_requests ADD COLUMN bytes_scanned_cap BIGINT; +ALTER TABLE query_requests ADD COLUMN bytes_scanned_cap_source bytes_scanned_cap_source; +ALTER TABLE query_requests ADD COLUMN bytes_scanned_cap_outcome bytes_scanned_cap_outcome; diff --git a/backend/src/main/resources/i18n/messages.properties b/backend/src/main/resources/i18n/messages.properties index 799f0c02d..f28bc607e 100644 --- a/backend/src/main/resources/i18n/messages.properties +++ b/backend/src/main/resources/i18n/messages.properties @@ -1583,3 +1583,24 @@ validation.row_limit_table.required=Target table is required validation.row_limit_table.size=Target table must be at most 255 characters validation.row_limit_max_rows.required=Max rows is required validation.row_limit_max_rows.range=Max rows must be between 1 and 1,000,000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Bytes-scanned cap must be at least 1 +error.bytes_scanned_cap_not_supported=Bytes-scanned caps are not supported for {0} datasources; they are available only for engines that report a bytes estimate (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Refused before execution: the estimated scan of {0} exceeds the bytes-scanned cap of {1} +error.bytes_cap.no_estimate_unreviewed=Refused before execution: no bytes estimate is available under the bytes-scanned cap of {0}, and no one approved this request group, so the review the cap requires never happened +error.bytes_cap.no_estimate=Refused before execution: no bytes estimate is available and this datasource rejects queries without one under its bytes-scanned cap of {0} +workflow.bytes_cap.rejected_exceeded=Rejected by the bytes-scanned cap: the estimated scan of {0} exceeds the cap of {1} +workflow.bytes_cap.rejected_no_estimate=Rejected by the bytes-scanned cap of {0}: no bytes estimate is available and this datasource rejects queries without one +workflow.decision.bytes_cap.none=No bytes-scanned cap applies +workflow.decision.bytes_cap.unevaluated=A bytes-scanned cap of {0} applies; a simulated request has no estimate to compare +workflow.decision.bytes_cap.within=The estimated scan of {0} is within the bytes-scanned cap of {1} +workflow.decision.bytes_cap.exceeded=The estimated scan of {0} exceeds the bytes-scanned cap of {1} — the request is rejected +workflow.decision.bytes_cap.no_estimate_review=No bytes estimate is available under the bytes-scanned cap of {0} — the request cannot auto-approve +workflow.decision.bytes_cap.no_estimate_rejected=No bytes estimate is available under the bytes-scanned cap of {0} — the request is rejected +workflow.decision.routing.skipped_bytes_cap=Routing policies are not evaluated when the bytes-scanned cap rejects the request +workflow.decision.grant.skipped_bytes_cap=The grant fast path is not evaluated when the bytes-scanned cap rejects the request +workflow.decision.plan.skipped_bytes_cap=The review plan is not consulted when the bytes-scanned cap rejects the request +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=Routing policy "{0}" matched with action AUTO_APPROVE, suppressed because no bytes estimate is available under the bytes-scanned cap +workflow.decision.grant.suppressed_bytes_cap=Covered by access grant {0}, suppressed because no bytes estimate is available under the bytes-scanned cap +workflow.decision.plan.suppressed_bytes_cap=The review plan would have approved, suppressed because no bytes estimate is available under the bytes-scanned cap diff --git a/backend/src/main/resources/i18n/messages_de.properties b/backend/src/main/resources/i18n/messages_de.properties index b928a04d1..77c24973e 100644 --- a/backend/src/main/resources/i18n/messages_de.properties +++ b/backend/src/main/resources/i18n/messages_de.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=Zieltabelle ist erforderlich validation.row_limit_table.size=Zieltabelle darf höchstens 255 Zeichen lang sein validation.row_limit_max_rows.required=Maximale Zeilen sind erforderlich validation.row_limit_max_rows.range=Maximale Zeilen müssen zwischen 1 und 1.000.000 liegen + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Das Limit für gescannte Bytes muss mindestens 1 sein +error.bytes_scanned_cap_not_supported=Limits für gescannte Bytes werden für {0}-Datenquellen nicht unterstützt; sie gibt es nur für Engines mit Byte-Schätzung (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Vor der Ausführung abgelehnt: der geschätzte Scan von {0} überschreitet das Limit für gescannte Bytes von {1} +error.bytes_cap.no_estimate_unreviewed=Vor der Ausführung abgelehnt: unter dem Limit für gescannte Bytes von {0} liegt keine Byte-Schätzung vor, und niemand hat diese Anfragegruppe genehmigt, daher hat die vom Limit verlangte Prüfung nie stattgefunden +error.bytes_cap.no_estimate=Vor der Ausführung abgelehnt: es liegt keine Byte-Schätzung vor, und diese Datenquelle lehnt Abfragen ohne Schätzung unter ihrem Limit von {0} ab +workflow.bytes_cap.rejected_exceeded=Vom Limit für gescannte Bytes abgelehnt: der geschätzte Scan von {0} überschreitet das Limit von {1} +workflow.bytes_cap.rejected_no_estimate=Vom Limit für gescannte Bytes von {0} abgelehnt: es liegt keine Byte-Schätzung vor, und diese Datenquelle lehnt Abfragen ohne Schätzung ab +workflow.decision.bytes_cap.none=Es gilt kein Limit für gescannte Bytes +workflow.decision.bytes_cap.unevaluated=Es gilt ein Limit für gescannte Bytes von {0}; eine simulierte Anfrage hat keine Schätzung zum Vergleich +workflow.decision.bytes_cap.within=Der geschätzte Scan von {0} liegt innerhalb des Limits für gescannte Bytes von {1} +workflow.decision.bytes_cap.exceeded=Der geschätzte Scan von {0} überschreitet das Limit für gescannte Bytes von {1} — die Anfrage wird abgelehnt +workflow.decision.bytes_cap.no_estimate_review=Unter dem Limit für gescannte Bytes von {0} liegt keine Byte-Schätzung vor — die Anfrage kann nicht automatisch genehmigt werden +workflow.decision.bytes_cap.no_estimate_rejected=Unter dem Limit für gescannte Bytes von {0} liegt keine Byte-Schätzung vor — die Anfrage wird abgelehnt +workflow.decision.routing.skipped_bytes_cap=Routing-Richtlinien werden nicht ausgewertet, wenn das Limit für gescannte Bytes die Anfrage ablehnt +workflow.decision.grant.skipped_bytes_cap=Der Grant-Schnellpfad wird nicht ausgewertet, wenn das Limit für gescannte Bytes die Anfrage ablehnt +workflow.decision.plan.skipped_bytes_cap=Der Prüfplan wird nicht herangezogen, wenn das Limit für gescannte Bytes die Anfrage ablehnt +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=Routing-Richtlinie "{0}" hat mit Aktion AUTO_APPROVE gegriffen, unterdrückt, weil unter dem Limit für gescannte Bytes keine Byte-Schätzung vorliegt +workflow.decision.grant.suppressed_bytes_cap=Durch Zugriffsgewährung {0} abgedeckt, unterdrückt, weil unter dem Limit für gescannte Bytes keine Byte-Schätzung vorliegt +workflow.decision.plan.suppressed_bytes_cap=Der Prüfplan hätte genehmigt, unterdrückt, weil unter dem Limit für gescannte Bytes keine Byte-Schätzung vorliegt diff --git a/backend/src/main/resources/i18n/messages_es.properties b/backend/src/main/resources/i18n/messages_es.properties index 58b1e2ff0..de158e904 100644 --- a/backend/src/main/resources/i18n/messages_es.properties +++ b/backend/src/main/resources/i18n/messages_es.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=La tabla de destino es obligatoria validation.row_limit_table.size=La tabla de destino debe tener como máximo 255 caracteres validation.row_limit_max_rows.required=El máximo de filas es obligatorio validation.row_limit_max_rows.range=El máximo de filas debe estar entre 1 y 1.000.000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=El límite de bytes escaneados debe ser al menos 1 +error.bytes_scanned_cap_not_supported=Los límites de bytes escaneados no se admiten en fuentes de datos {0}; solo están disponibles para motores que informan una estimación de bytes (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Rechazada antes de ejecutarse: el escaneo estimado de {0} supera el límite de bytes escaneados de {1} +error.bytes_cap.no_estimate_unreviewed=Rechazada antes de ejecutarse: no hay estimación de bytes bajo el límite de bytes escaneados de {0} y nadie aprobó este grupo de solicitudes, así que la revisión que exige el límite nunca ocurrió +error.bytes_cap.no_estimate=Rechazada antes de ejecutarse: no hay estimación de bytes y esta fuente de datos rechaza las consultas sin ella bajo su límite de {0} +workflow.bytes_cap.rejected_exceeded=Rechazada por el límite de bytes escaneados: el escaneo estimado de {0} supera el límite de {1} +workflow.bytes_cap.rejected_no_estimate=Rechazada por el límite de bytes escaneados de {0}: no hay estimación de bytes y esta fuente de datos rechaza las consultas sin ella +workflow.decision.bytes_cap.none=No se aplica ningún límite de bytes escaneados +workflow.decision.bytes_cap.unevaluated=Se aplica un límite de bytes escaneados de {0}; una solicitud simulada no tiene estimación para comparar +workflow.decision.bytes_cap.within=El escaneo estimado de {0} está dentro del límite de bytes escaneados de {1} +workflow.decision.bytes_cap.exceeded=El escaneo estimado de {0} supera el límite de bytes escaneados de {1} — la solicitud se rechaza +workflow.decision.bytes_cap.no_estimate_review=No hay estimación de bytes bajo el límite de bytes escaneados de {0} — la solicitud no puede aprobarse automáticamente +workflow.decision.bytes_cap.no_estimate_rejected=No hay estimación de bytes bajo el límite de bytes escaneados de {0} — la solicitud se rechaza +workflow.decision.routing.skipped_bytes_cap=Las políticas de enrutamiento no se evalúan cuando el límite de bytes escaneados rechaza la solicitud +workflow.decision.grant.skipped_bytes_cap=La vía rápida por concesión no se evalúa cuando el límite de bytes escaneados rechaza la solicitud +workflow.decision.plan.skipped_bytes_cap=El plan de revisión no se consulta cuando el límite de bytes escaneados rechaza la solicitud +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=La política de enrutamiento "{0}" coincidió con la acción AUTO_APPROVE, suprimida porque no hay estimación de bytes bajo el límite de bytes escaneados +workflow.decision.grant.suppressed_bytes_cap=Cubierta por la concesión de acceso {0}, suprimida porque no hay estimación de bytes bajo el límite de bytes escaneados +workflow.decision.plan.suppressed_bytes_cap=El plan de revisión habría aprobado, suprimido porque no hay estimación de bytes bajo el límite de bytes escaneados diff --git a/backend/src/main/resources/i18n/messages_fr.properties b/backend/src/main/resources/i18n/messages_fr.properties index 25881f4a1..b072b3a59 100644 --- a/backend/src/main/resources/i18n/messages_fr.properties +++ b/backend/src/main/resources/i18n/messages_fr.properties @@ -1568,3 +1568,24 @@ validation.row_limit_table.required=La table cible est obligatoire validation.row_limit_table.size=La table cible ne doit pas dépasser 255 caractères validation.row_limit_max_rows.required=Le nombre maximal de lignes est obligatoire validation.row_limit_max_rows.range=Le nombre maximal de lignes doit être compris entre 1 et 1 000 000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=La limite d’octets analysés doit être d’au moins 1 +error.bytes_scanned_cap_not_supported=Les limites d’octets analysés ne sont pas prises en charge pour les sources de données {0} ; elles sont réservées aux moteurs qui fournissent une estimation en octets (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Refusée avant exécution : le volume analysé estimé de {0} dépasse la limite d’octets analysés de {1} +error.bytes_cap.no_estimate_unreviewed=Refusée avant exécution : aucune estimation en octets n’est disponible sous la limite d’octets analysés de {0}, et personne n’a approuvé ce groupe de demandes ; la revue exigée par la limite n’a donc jamais eu lieu +error.bytes_cap.no_estimate=Refusée avant exécution : aucune estimation en octets n’est disponible et cette source de données refuse les requêtes sans estimation sous sa limite de {0} +workflow.bytes_cap.rejected_exceeded=Rejetée par la limite d’octets analysés : le volume analysé estimé de {0} dépasse la limite de {1} +workflow.bytes_cap.rejected_no_estimate=Rejetée par la limite d’octets analysés de {0} : aucune estimation en octets n’est disponible et cette source de données refuse les requêtes sans estimation +workflow.decision.bytes_cap.none=Aucune limite d’octets analysés ne s’applique +workflow.decision.bytes_cap.unevaluated=Une limite d’octets analysés de {0} s’applique ; une demande simulée n’a pas d’estimation à comparer +workflow.decision.bytes_cap.within=Le volume analysé estimé de {0} respecte la limite d’octets analysés de {1} +workflow.decision.bytes_cap.exceeded=Le volume analysé estimé de {0} dépasse la limite d’octets analysés de {1} — la demande est rejetée +workflow.decision.bytes_cap.no_estimate_review=Aucune estimation en octets sous la limite d’octets analysés de {0} — la demande ne peut pas être approuvée automatiquement +workflow.decision.bytes_cap.no_estimate_rejected=Aucune estimation en octets sous la limite d’octets analysés de {0} — la demande est rejetée +workflow.decision.routing.skipped_bytes_cap=Les politiques de routage ne sont pas évaluées lorsque la limite d’octets analysés rejette la demande +workflow.decision.grant.skipped_bytes_cap=Le raccourci par autorisation n’est pas évalué lorsque la limite d’octets analysés rejette la demande +workflow.decision.plan.skipped_bytes_cap=Le plan de revue n’est pas consulté lorsque la limite d’octets analysés rejette la demande +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=La politique de routage "{0}" a correspondu avec l’action AUTO_APPROVE, neutralisée faute d’estimation en octets sous la limite d’octets analysés +workflow.decision.grant.suppressed_bytes_cap=Couverte par l’autorisation d’accès {0}, neutralisée faute d’estimation en octets sous la limite d’octets analysés +workflow.decision.plan.suppressed_bytes_cap=Le plan de revue aurait approuvé, neutralisé faute d’estimation en octets sous la limite d’octets analysés diff --git a/backend/src/main/resources/i18n/messages_hy.properties b/backend/src/main/resources/i18n/messages_hy.properties index fc51435bf..deb3b475b 100644 --- a/backend/src/main/resources/i18n/messages_hy.properties +++ b/backend/src/main/resources/i18n/messages_hy.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=Թիրախ աղյուսակը պարտադի validation.row_limit_table.size=Թիրախ աղյուսակը պետք է լինի առավելագույնը 255 նիշ validation.row_limit_max_rows.required=Տողերի առավելագույն քանակը պարտադիր է validation.row_limit_max_rows.range=Տողերի առավելագույն քանակը պետք է լինի 1-ից 1 000 000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Սկանավորված բայթերի սահմանաչափը պետք է լինի առնվազն 1 +error.bytes_scanned_cap_not_supported=Սկանավորված բայթերի սահմանաչափերը չեն աջակցվում {0} տվյալների աղբյուրների համար. դրանք հասանելի են միայն բայթերի գնահատական տրամադրող շարժիչների համար (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Մերժվել է կատարումից առաջ. {0} գնահատված սկանավորումը գերազանցում է {1} սկանավորված բայթերի սահմանաչափը +error.bytes_cap.no_estimate_unreviewed=Մերժվել է կատարումից առաջ. {0} սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա, և ոչ ոք չի հաստատել հարցումների այս խումբը, ուստի սահմանաչափի պահանջած ստուգումը տեղի չի ունեցել +error.bytes_cap.no_estimate=Մերժվել է կատարումից առաջ. բայթերի գնահատական չկա, և այս տվյալների աղբյուրը {0} սահմանաչափի ներքո մերժում է առանց գնահատականի հարցումները +workflow.bytes_cap.rejected_exceeded=Մերժվել է սկանավորված բայթերի սահմանաչափով. {0} գնահատված սկանավորումը գերազանցում է {1} սահմանաչափը +workflow.bytes_cap.rejected_no_estimate=Մերժվել է {0} սկանավորված բայթերի սահմանաչափով. բայթերի գնահատական չկա, և այս տվյալների աղբյուրը մերժում է առանց գնահատականի հարցումները +workflow.decision.bytes_cap.none=Սկանավորված բայթերի սահմանաչափ չի կիրառվում +workflow.decision.bytes_cap.unevaluated=Կիրառվում է {0} սկանավորված բայթերի սահմանաչափ. մոդելավորված հարցումը համեմատելու գնահատական չունի +workflow.decision.bytes_cap.within={0} գնահատված սկանավորումը {1} սկանավորված բայթերի սահմանաչափի մեջ է +workflow.decision.bytes_cap.exceeded={0} գնահատված սկանավորումը գերազանցում է {1} սկանավորված բայթերի սահմանաչափը — հարցումը մերժվում է +workflow.decision.bytes_cap.no_estimate_review={0} սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա — հարցումը չի կարող ավտոմատ հաստատվել +workflow.decision.bytes_cap.no_estimate_rejected={0} սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա — հարցումը մերժվում է +workflow.decision.routing.skipped_bytes_cap=Երթուղավորման քաղաքականությունները չեն գնահատվում, երբ սկանավորված բայթերի սահմանաչափը մերժում է հարցումը +workflow.decision.grant.skipped_bytes_cap=Թույլտվության արագ ուղին չի գնահատվում, երբ սկանավորված բայթերի սահմանաչափը մերժում է հարցումը +workflow.decision.plan.skipped_bytes_cap=Ստուգման պլանը չի դիտարկվում, երբ սկանավորված բայթերի սահմանաչափը մերժում է հարցումը +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=«{0}» երթուղավորման քաղաքականությունը համընկավ AUTO_APPROVE գործողությամբ, ճնշվել է, քանի որ սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա +workflow.decision.grant.suppressed_bytes_cap=Ծածկված է {0} մուտքի թույլտվությամբ, ճնշվել է, քանի որ սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա +workflow.decision.plan.suppressed_bytes_cap=Ստուգման պլանը կհաստատեր, ճնշվել է, քանի որ սկանավորված բայթերի սահմանաչափի ներքո բայթերի գնահատական չկա diff --git a/backend/src/main/resources/i18n/messages_ru.properties b/backend/src/main/resources/i18n/messages_ru.properties index 13d462ce9..ccadf8a29 100644 --- a/backend/src/main/resources/i18n/messages_ru.properties +++ b/backend/src/main/resources/i18n/messages_ru.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=Целевая таблица обязате validation.row_limit_table.size=Целевая таблица не должна превышать 255 символов validation.row_limit_max_rows.required=Максимальное число строк обязательно validation.row_limit_max_rows.range=Максимальное число строк должно быть от 1 до 1 000 000 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=Лимит сканируемых байтов должен быть не менее 1 +error.bytes_scanned_cap_not_supported=Лимиты сканируемых байтов не поддерживаются для источников данных {0}; они доступны только для движков, сообщающих оценку в байтах (BigQuery, Snowflake, Databricks) +error.bytes_cap.exceeded=Отклонено перед выполнением: оценочный объём сканирования {0} превышает лимит сканируемых байтов {1} +error.bytes_cap.no_estimate_unreviewed=Отклонено перед выполнением: при лимите сканируемых байтов {0} оценка в байтах недоступна, а эту группу запросов никто не одобрил, поэтому требуемая лимитом проверка не проводилась +error.bytes_cap.no_estimate=Отклонено перед выполнением: оценка в байтах недоступна, а этот источник данных отклоняет запросы без неё при лимите {0} +workflow.bytes_cap.rejected_exceeded=Отклонено лимитом сканируемых байтов: оценочный объём сканирования {0} превышает лимит {1} +workflow.bytes_cap.rejected_no_estimate=Отклонено лимитом сканируемых байтов {0}: оценка в байтах недоступна, а этот источник данных отклоняет запросы без неё +workflow.decision.bytes_cap.none=Лимит сканируемых байтов не применяется +workflow.decision.bytes_cap.unevaluated=Применяется лимит сканируемых байтов {0}; у смоделированного запроса нет оценки для сравнения +workflow.decision.bytes_cap.within=Оценочный объём сканирования {0} укладывается в лимит сканируемых байтов {1} +workflow.decision.bytes_cap.exceeded=Оценочный объём сканирования {0} превышает лимит сканируемых байтов {1} — запрос отклонён +workflow.decision.bytes_cap.no_estimate_review=При лимите сканируемых байтов {0} оценка в байтах недоступна — запрос не может быть одобрен автоматически +workflow.decision.bytes_cap.no_estimate_rejected=При лимите сканируемых байтов {0} оценка в байтах недоступна — запрос отклонён +workflow.decision.routing.skipped_bytes_cap=Политики маршрутизации не оцениваются, когда лимит сканируемых байтов отклоняет запрос +workflow.decision.grant.skipped_bytes_cap=Быстрый путь по выдаче доступа не оценивается, когда лимит сканируемых байтов отклоняет запрос +workflow.decision.plan.skipped_bytes_cap=План проверки не учитывается, когда лимит сканируемых байтов отклоняет запрос +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=Политика маршрутизации "{0}" сработала с действием AUTO_APPROVE, подавлено, так как при лимите сканируемых байтов оценка в байтах недоступна +workflow.decision.grant.suppressed_bytes_cap=Покрыт выдачей доступа {0}, подавлено, так как при лимите сканируемых байтов оценка в байтах недоступна +workflow.decision.plan.suppressed_bytes_cap=План проверки одобрил бы запрос, подавлено, так как при лимите сканируемых байтов оценка в байтах недоступна diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index f5fe78f60..9fff29495 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -1562,3 +1562,24 @@ validation.row_limit_table.required=目标表为必填项 validation.row_limit_table.size=目标表最多 255 个字符 validation.row_limit_max_rows.required=最大行数为必填项 validation.row_limit_max_rows.range=最大行数必须在 1 到 1,000,000 之间 + +# Bytes-scanned cost caps (#941) +validation.bytes_cap.min=扫描字节上限至少为 1 +error.bytes_scanned_cap_not_supported={0} 数据源不支持扫描字节上限；仅适用于提供字节估算的引擎（BigQuery、Snowflake、Databricks） +error.bytes_cap.exceeded=执行前被拒绝：预计扫描量 {0} 超过扫描字节上限 {1} +error.bytes_cap.no_estimate_unreviewed=执行前被拒绝：在 {0} 的扫描字节上限下没有可用的字节估算，且没有人批准此请求组，因此上限要求的审查从未进行 +error.bytes_cap.no_estimate=执行前被拒绝：没有可用的字节估算，且该数据源在 {0} 的扫描字节上限下拒绝没有估算的查询 +workflow.bytes_cap.rejected_exceeded=被扫描字节上限拒绝：预计扫描量 {0} 超过上限 {1} +workflow.bytes_cap.rejected_no_estimate=被 {0} 的扫描字节上限拒绝：没有可用的字节估算，且该数据源拒绝没有估算的查询 +workflow.decision.bytes_cap.none=未应用扫描字节上限 +workflow.decision.bytes_cap.unevaluated=适用 {0} 的扫描字节上限；模拟请求没有可比较的估算 +workflow.decision.bytes_cap.within=预计扫描量 {0} 在扫描字节上限 {1} 之内 +workflow.decision.bytes_cap.exceeded=预计扫描量 {0} 超过扫描字节上限 {1} — 请求被拒绝 +workflow.decision.bytes_cap.no_estimate_review=在 {0} 的扫描字节上限下没有可用的字节估算 — 请求无法自动批准 +workflow.decision.bytes_cap.no_estimate_rejected=在 {0} 的扫描字节上限下没有可用的字节估算 — 请求被拒绝 +workflow.decision.routing.skipped_bytes_cap=扫描字节上限拒绝请求时不评估路由策略 +workflow.decision.grant.skipped_bytes_cap=扫描字节上限拒绝请求时不评估授权快速通道 +workflow.decision.plan.skipped_bytes_cap=扫描字节上限拒绝请求时不参考审查计划 +workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap=路由策略"{0}"以 AUTO_APPROVE 动作匹配，但因在扫描字节上限下没有可用的字节估算而被抑制 +workflow.decision.grant.suppressed_bytes_cap=由访问授权 {0} 覆盖，但因在扫描字节上限下没有可用的字节估算而被抑制 +workflow.decision.plan.suppressed_bytes_cap=审查计划本会批准，但因在扫描字节上限下没有可用的字节估算而被抑制 diff --git a/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java b/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java index 27bad6809..85696530a 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/access/internal/AccessGrantMaterializerTest.java @@ -69,7 +69,7 @@ private AccessGrantRequestEntity approved() { private DatasourcePermissionView granted() { return new DatasourcePermissionView(newPermissionId, datasourceId, requesterId, "u@x.io", - "U", true, false, false, false, null, List.of("public"), null, null, null, List.of(), List.of(), + "U", true, false, false, false, null, null, List.of("public"), null, null, null, List.of(), List.of(), List.of(), Instant.now().plusSeconds(3600), approverId, Instant.now()); } @@ -155,6 +155,25 @@ void materialiseCarriesTheReplacedRowsDenialsOntoTheNewGrant() { .containsExactly(com.bablsoft.accessflow.core.api.QueryShape.JOIN); } + @Test + void materialiseCarriesTheReplacedRowsBytesScannedCapSoAJitApprovalNeverWidensIt() { + when(requestRepository.findById(requestId)).thenReturn(Optional.of(approved())); + var existing = new DatasourceUserPermissionView(UUID.randomUUID(), requesterId, + datasourceId, true, false, false, false, null, null, null, null, null, null, null, + null, 4_000L, Instant.now().plusSeconds(60)); + when(permissionLookupService.findDirectFor(requesterId, datasourceId)) + .thenReturn(Optional.of(existing)); + when(datasourceAdminService.grantPermission(any(), any(), any(), any())) + .thenReturn(granted()); + + materializer.materialize(requestId, approverId); + + var captor = ArgumentCaptor.forClass(CreatePermissionCommand.class); + verify(datasourceAdminService).grantPermission(eq(datasourceId), eq(organizationId), + eq(approverId), captor.capture()); + assertThat(captor.getValue().bytesScannedLimitOverride()).isEqualTo(4_000L); + } + @Test void materialiseWithoutAnExistingRowCarriesNoDenials() { when(requestRepository.findById(requestId)).thenReturn(Optional.of(approved())); diff --git a/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java index 4aa4ac2a6..6f406ce6b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/access/internal/DefaultGrantUsageAggregationServiceTest.java @@ -109,7 +109,7 @@ private void givenDatasourceGrant(List allowedTables, Instant grantedAt) .thenReturn(List.of(new DatasourceRef(DATASOURCE, "analytics"))); when(datasourceAdminService.listPermissions(DATASOURCE, ORG)).thenReturn(List.of( new DatasourcePermissionView(PERMISSION, DATASOURCE, USER, "dev@example.test", - "Dev", true, false, false, false, null, List.of(), allowedTables, List.of(), null, List.of(), List.of(), + "Dev", true, false, false, false, null, null, List.of(), allowedTables, List.of(), null, List.of(), List.of(), List.of(), null, UUID.randomUUID(), grantedAt))); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java index 3a7fa7b5e..9c9db2c6a 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/ai/internal/DefaultAiAnalyzerServiceTest.java @@ -251,6 +251,28 @@ void analyzeSubmittedQueryPersistsResultAndPublishesCompleted() { .isEqualTo(50.0); } + @Test + void theCostEstimateContextCarriesTheWarehouseBytesEstimate() { + var snapshot = new QueryRequestSnapshot(queryRequestId, datasourceId, organizationId, userId, + "SELECT 1", QueryType.SELECT, false, QueryStatus.PENDING_AI, null, null, null, false); + when(queryRequestLookupService.findById(queryRequestId)).thenReturn(Optional.of(snapshot)); + when(datasourceLookupService.findById(datasourceId)).thenReturn(Optional.of(descriptor(DbType.MYSQL))); + when(datasourceAdminService.introspectSchemaForSystem(datasourceId, organizationId)).thenReturn(schemaView()); + when(queryCostEstimateService.estimateSubmittedQuery(queryRequestId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), + queryRequestId, "bigquery", QueryType.SELECT, true, null, null, null, null, + 2_500_000_000L, null, null, null, false, null, 5, java.time.Instant.now()))); + var costContext = ArgumentCaptor.forClass(String.class); + when(strategy.analyze(eq("SELECT 1"), eq(DbType.MYSQL), any(), costContext.capture(), any(), + eq(aiConfigId))).thenReturn(sampleResult()); + when(aiAnalysisPersistenceService.persist(eq(queryRequestId), any())).thenReturn(UUID.randomUUID()); + + service.analyzeSubmittedQuery(queryRequestId); + + assertThat(costContext.getValue()) + .isEqualTo("The warehouse estimates this query will scan 2.5 GB (2500000000 bytes)."); + } + @Test void analyzeSubmittedQueryPersistsPerModelBreakdown() { var snapshot = new QueryRequestSnapshot(queryRequestId, datasourceId, organizationId, userId, diff --git a/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java index 22fbde2d5..a397125ed 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/attestation/AttestationLifecycleIntegrationTest.java @@ -130,7 +130,7 @@ private UserEntity user(String name, UserRoleType role) { private void grant(UserEntity subject) { datasourceAdminService.grantPermission(datasource.getId(), organization.getId(), admin.getId(), new CreatePermissionCommand(subject.getId(), true, false, false, - false, null, List.of("public"), null, null, null, null, null, List.of(), null, null)); + false, null, null, List.of("public"), null, null, null, null, null, List.of(), null, null)); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java index e3d149031..92b482f37 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/attestation/internal/DefaultAttestationLifecycleServiceTest.java @@ -79,7 +79,7 @@ private AttestationCampaignEntity scheduledDatasourceCampaign() { private DatasourcePermissionView permission(UUID userId) { return new DatasourcePermissionView(UUID.randomUUID(), datasourceId, userId, userId + "@example.com", "User", true, false, false, false, null, - List.of("public"), List.of(), List.of(), null, List.of(), List.of(), List.of(), null, UUID.randomUUID(), Instant.now()); + null, List.of("public"), List.of(), List.of(), null, List.of(), List.of(), List.of(), null, UUID.randomUUID(), Instant.now()); } @Test @@ -113,7 +113,7 @@ void openSnapshotsTheGrantsTableAndSchemaDenials() { when(datasourceAdminService.listPermissions(datasourceId, orgId)) .thenReturn(List.of(new DatasourcePermissionView(UUID.randomUUID(), datasourceId, userId, "u@example.com", "User", true, false, false, false, null, - List.of("crm"), List.of(), List.of(), List.of(), List.of("hr"), + 8_000L, List.of("crm"), List.of(), List.of(), List.of(), List.of("hr"), List.of("crm.salary"), List.of(com.bablsoft.accessflow.core.api.QueryShape.GROUP_BY), null, UUID.randomUUID(), Instant.now()))); @@ -129,6 +129,7 @@ void openSnapshotsTheGrantsTableAndSchemaDenials() { assertThat(snapshot.get("denied_tables").get(0).asString()).isEqualTo("crm.salary"); assertThat(snapshot.get("denied_tables")).hasSize(1); assertThat(snapshot.get("denied_shapes").get(0).asString()).isEqualTo("GROUP_BY"); + assertThat(snapshot.get("bytes_scanned_limit_override").asLong()).isEqualTo(8_000L); } @Test @@ -138,7 +139,7 @@ void openSnapshotsEmptyDenialArraysForAGrantWithoutDenials() { when(datasourceLookupService.findRef(datasourceId)) .thenReturn(Optional.of(new DatasourceRef(datasourceId, "Production"))); var view = new DatasourcePermissionView(UUID.randomUUID(), datasourceId, UUID.randomUUID(), - "u@example.com", "User", true, false, false, false, null, null, null, null, null, + "u@example.com", "User", true, false, false, false, null, null, null, null, null, null, null, null, null, null, null, null); when(datasourceAdminService.listPermissions(datasourceId, orgId)).thenReturn(List.of(view)); when(itemRepository.existsByCampaignIdAndPermissionId(any(), any())).thenReturn(false); @@ -152,6 +153,7 @@ void openSnapshotsEmptyDenialArraysForAGrantWithoutDenials() { assertThat(snapshot.get("denied_schemas")).isEmpty(); assertThat(snapshot.get("denied_tables")).isEmpty(); assertThat(snapshot.get("denied_shapes")).isEmpty(); + assertThat(snapshot.get("bytes_scanned_limit_override").isNull()).isTrue(); } /** diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java new file mode 100644 index 000000000..a5151fca9 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/core/api/BytesScannedCapRecordsTest.java @@ -0,0 +1,90 @@ +package com.bablsoft.accessflow.core.api; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class BytesScannedCapRecordsTest { + + @Test + void anEstimateAboveTheCapExceedsIt() { + var cap = new AppliedBytesCap(100, BytesScannedCapSource.DATASOURCE, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + assertThat(cap.check(101L)).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + assertThat(cap.check(100L)).isEqualTo(BytesScannedCapOutcome.WITHIN); + assertThat(cap.check(0L)).isEqualTo(BytesScannedCapOutcome.WITHIN); + } + + @Test + void aMissingEstimateFollowsTheDatasourcePolicy() { + var review = new AppliedBytesCap(100, BytesScannedCapSource.GRANT, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var reject = new AppliedBytesCap(100, BytesScannedCapSource.GRANT, + BytesCapMissingEstimateAction.REJECT); + + assertThat(review.check(null)).isEqualTo(BytesScannedCapOutcome.NO_ESTIMATE_REVIEW); + assertThat(reject.check(null)).isEqualTo(BytesScannedCapOutcome.NO_ESTIMATE_REJECTED); + } + + @Test + void anAbsentPolicyDefaultsToRequireReview() { + var cap = new AppliedBytesCap(1, BytesScannedCapSource.DATASOURCE, null); + + assertThat(cap.missingEstimate()).isEqualTo(BytesCapMissingEstimateAction.REQUIRE_REVIEW); + } + + @Test + void aCapMustBePositiveAndNamed() { + assertThatThrownBy(() -> new AppliedBytesCap(0, BytesScannedCapSource.DATASOURCE, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new AppliedBytesCap(1, null, null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void onlyTheTwoRefusingOutcomesReject() { + assertThat(BytesScannedCapOutcome.EXCEEDED.rejects()).isTrue(); + assertThat(BytesScannedCapOutcome.NO_ESTIMATE_REJECTED.rejects()).isTrue(); + assertThat(BytesScannedCapOutcome.WITHIN.rejects()).isFalse(); + assertThat(BytesScannedCapOutcome.NO_ESTIMATE_REVIEW.rejects()).isFalse(); + } + + @Test + void onlyTheBytesReportingWarehousesSupportACap() { + assertThat(BytesScannedCapSupport.supports(DbType.BIGQUERY)).isTrue(); + assertThat(BytesScannedCapSupport.supports(DbType.SNOWFLAKE)).isTrue(); + assertThat(BytesScannedCapSupport.supports(DbType.DATABRICKS)).isTrue(); + assertThat(BytesScannedCapSupport.supports(DbType.POSTGRESQL)).isFalse(); + assertThat(BytesScannedCapSupport.supports(DbType.MONGODB)).isFalse(); + assertThat(BytesScannedCapSupport.supports(null)).isFalse(); + assertThat(BytesScannedCapSupport.supportedTypes()).hasSize(3); + } + + @Test + void theExceptionsCarryWhatTheirHandlersRender() { + var notSupported = new BytesScannedCapNotSupportedException(DbType.POSTGRESQL); + assertThat(notSupported.dbType()).isEqualTo(DbType.POSTGRESQL); + assertThat(notSupported).hasMessageContaining("POSTGRESQL"); + + var cap = new AppliedBytesCap(10, BytesScannedCapSource.GRANT, null); + var exceeded = new BytesScannedCapExceededException("too big", cap, 20L, + BytesScannedCapOutcome.EXCEEDED); + assertThat(exceeded).hasMessage("too big"); + assertThat(exceeded.cap()).isEqualTo(cap); + assertThat(exceeded.estimatedBytes()).isEqualTo(20L); + assertThat(exceeded.outcome()).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + } + + @Test + void byteSizesRenderInDecimalUnitsWithTheExactFigure() { + assertThat(ByteSizeFormat.format(0)).isEqualTo("0 B"); + assertThat(ByteSizeFormat.format(999)).isEqualTo("999 B"); + assertThat(ByteSizeFormat.format(1_000)).isEqualTo("1 KB (1000 B)"); + assertThat(ByteSizeFormat.format(1_500_000_000_000L)) + .isEqualTo("1.5 TB (1500000000000 B)"); + assertThat(ByteSizeFormat.format(1_234_567_890L)).isEqualTo("1.23 GB (1234567890 B)"); + assertThat(ByteSizeFormat.format(Long.MAX_VALUE)).startsWith("9.22 EB"); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java index db0c4ca80..85a450973 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DatasourceAdminServiceImplTest.java @@ -830,6 +830,121 @@ void updateClearEnvironmentUnsetsItAndAnExplicitValueWins() { assertThat(entity.getEnvironment()).isEqualTo(DatasourceEnvironment.DEVELOPMENT); } + // ── Bytes-scanned cap (#941) ────────────────────────────────────────────── + + @Test + void createStoresABytesScannedCapOnAWarehouse() { + var org = new OrganizationEntity(); + org.setId(orgId); + when(organizationRepository.getReferenceById(orgId)).thenReturn(org); + when(encryptionService.encrypt("pw")).thenReturn("ENC(pw)"); + when(engineCatalog.isEngineManaged(DbType.SNOWFLAKE)).thenReturn(true); + when(datasourceRepository.save(any(DatasourceEntity.class))) + .thenAnswer(inv -> inv.getArgument(0)); + + var result = service.create(bytesCapCreateCommand(DbType.SNOWFLAKE, 1_000_000L, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT)); + + assertThat(result.maxBytesScannedPerQuery()).isEqualTo(1_000_000L); + assertThat(result.bytesCapMissingEstimate()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + } + + @Test + void createWithoutACapDefaultsToRequireReview() { + var org = new OrganizationEntity(); + org.setId(orgId); + when(organizationRepository.getReferenceById(orgId)).thenReturn(org); + when(encryptionService.encrypt("pw")).thenReturn("ENC(pw)"); + when(datasourceRepository.save(any(DatasourceEntity.class))) + .thenAnswer(inv -> inv.getArgument(0)); + + var result = service.create(createCommand("Prod", null)); + + assertThat(result.maxBytesScannedPerQuery()).isNull(); + assertThat(result.bytesCapMissingEstimate()).isEqualTo( + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + } + + @Test + void createRefusesABytesScannedCapOnAnEngineWithoutABytesEstimate() { + var org = new OrganizationEntity(); + org.setId(orgId); + when(organizationRepository.getReferenceById(orgId)).thenReturn(org); + when(encryptionService.encrypt("pw")).thenReturn("ENC(pw)"); + + assertThatThrownBy(() -> service.create(bytesCapCreateCommand(DbType.POSTGRESQL, 10L, null))) + .isInstanceOf(com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException.class); + verify(datasourceRepository, never()).save(any()); + } + + @Test + void updateSetsClearsAndGatesTheBytesScannedCap() { + var entity = buildDatasource(datasourceId, orgId, "Wh"); + entity.setDbType(DbType.BIGQUERY); + when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.of(entity)); + + service.update(datasourceId, orgId, bytesCapUpdateCommand(500L, null, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT)); + assertThat(entity.getMaxBytesScannedPerQuery()).isEqualTo(500L); + assertThat(entity.getBytesCapMissingEstimate()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + + service.update(datasourceId, orgId, bytesCapUpdateCommand(null, null, null)); + assertThat(entity.getMaxBytesScannedPerQuery()).isEqualTo(500L); + + service.update(datasourceId, orgId, bytesCapUpdateCommand(null, true, null)); + assertThat(entity.getMaxBytesScannedPerQuery()).isNull(); + + entity.setDbType(DbType.MYSQL); + assertThatThrownBy(() -> service.update(datasourceId, orgId, + bytesCapUpdateCommand(10L, null, null))) + .isInstanceOf(com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException.class); + } + + @Test + void grantPermissionStoresABytesScannedOverrideOnlyOnAWarehouse() { + stubGrantableUser(DbType.DATABRICKS); + var saved = ArgumentCaptor.forClass(DatasourceUserPermissionEntity.class); + when(permissionRepository.save(saved.capture())).thenAnswer(inv -> inv.getArgument(0)); + + var view = service.grantPermission(datasourceId, orgId, adminId, bytesCapPermission(900L)); + + assertThat(saved.getValue().getBytesScannedLimitOverride()).isEqualTo(900L); + assertThat(view.bytesScannedLimitOverride()).isEqualTo(900L); + } + + @Test + void grantPermissionRefusesABytesScannedOverrideOnARelationalDatasource() { + stubGrantableUser(DbType.POSTGRESQL); + + assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, + bytesCapPermission(900L))) + .isInstanceOf(com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException.class); + verify(permissionRepository, never()).save(any()); + } + + private CreateDatasourceCommand bytesCapCreateCommand( + DbType dbType, Long cap, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + return new CreateDatasourceCommand(orgId, "Wh", dbType, "wh.example.com", 443, "ANALYTICS", + "svc", "pw", SslMode.REQUIRE, null, null, null, null, null, false, null, null, + null, null, null, null, null, null, null, null, null, null, cap, missing); + } + + private static UpdateDatasourceCommand bytesCapUpdateCommand( + Long cap, Boolean clear, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + return new UpdateDatasourceCommand(null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, cap, clear, missing); + } + + private CreatePermissionCommand bytesCapPermission(Long override) { + return new CreatePermissionCommand(userId, true, false, false, false, null, override, null, + null, null, null, null, null, null, null, null); + } + private CreateDatasourceCommand createCommand(String name, DatasourceEnvironment environment) { return new CreateDatasourceCommand(orgId, name, DbType.POSTGRESQL, "db", 5432, "appdb", "svc", "pw", SslMode.DISABLE, null, null, null, null, null, false, null, null, @@ -1076,7 +1191,7 @@ void grantPermissionRejectsUserFromDifferentOrg() { user.setOrganization(otherOrg); when(userRepository.findById(userId)).thenReturn(Optional.of(user)); - var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, + var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null, null); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(IllegalDatasourcePermissionException.class); @@ -1088,7 +1203,7 @@ void grantPermissionRejectsUnknownUser() { when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.of(entity)); when(userRepository.findById(userId)).thenReturn(Optional.empty()); - var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, + var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null, null); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(IllegalDatasourcePermissionException.class); @@ -1107,7 +1222,7 @@ void grantPermissionRejectsDuplicate() { when(permissionRepository.existsByUser_IdAndDatasource_Id(userId, datasourceId)) .thenReturn(true); - var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, + var command = new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null, null); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(DatasourcePermissionAlreadyExistsException.class); @@ -1135,7 +1250,7 @@ void grantPermissionPersistsAndReturnsView() { .thenAnswer(inv -> inv.getArgument(0)); var command = new CreatePermissionCommand(userId, true, true, false, true, 500, - List.of("public"), List.of("orders"), List.of("public.orders.ssn"), null, null, null, List.of(), null, null); + null, List.of("public"), List.of("orders"), List.of("public.orders.ssn"), null, null, null, List.of(), null, null); var view = service.grantPermission(datasourceId, orgId, adminId, command); // An admin-created row has no originating JIT request (#969). @@ -1174,7 +1289,7 @@ void grantPermissionStampsTheOriginatingAccessRequest() { var accessGrantRequestId = UUID.randomUUID(); service.grantPermission(datasourceId, orgId, adminId, new CreatePermissionCommand(userId, - true, false, false, false, null, null, null, null, null, null, null, + true, false, false, false, null, null, null, null, null, null, null, null, List.of(), Instant.now().plusSeconds(3600), accessGrantRequestId)); @@ -1240,7 +1355,7 @@ void grantPermissionPersistsNormalizedDeniedSchemasAndTables() { var view = service.grantPermission(datasourceId, orgId, adminId, new CreatePermissionCommand(userId, true, false, false, false, null, - List.of("crm"), null, null, null, List.of(" \"HR\" ", "hr"), + null, List.of("crm"), null, null, null, List.of(" \"HR\" ", "hr"), List.of("CRM.Salary", "`crm`.`salary`", "bonus"), List.of(), null, null)); assertThat(saved.getValue().getDeniedSchemas()).containsExactly("hr"); @@ -1254,12 +1369,12 @@ void grantPermissionRefusesADeniedEntryThatCouldNeverMatch() { stubGrantableUser(DbType.POSTGRESQL); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, - new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, List.of("analytics.hr"), null, List.of(), null, null))) .isInstanceOf(IllegalDatasourcePermissionException.class) .hasMessageContaining("denied_schemas"); assertThatThrownBy(() -> service.grantPermission(datasourceId, orgId, adminId, - new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, List.of("sal*"), List.of(), null, null))) .isInstanceOf(IllegalDatasourcePermissionException.class) .hasMessageContaining("denied_tables"); @@ -1273,7 +1388,7 @@ void grantPermissionStoresNoDeniedSchemasOrTablesWhenEmptyOrBlank() { when(permissionRepository.save(saved.capture())).thenAnswer(inv -> inv.getArgument(0)); var view = service.grantPermission(datasourceId, orgId, adminId, - new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, List.of(), List.of(" "), List.of(), null, null)); assertThat(saved.getValue().getDeniedSchemas()).isNull(); @@ -1321,7 +1436,7 @@ void grantPermissionRejectsDeniedShapesOnAnEngineManagedDatasource() { } private CreatePermissionCommand shapesCommand(List deniedShapes) { - return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, null, null, null, deniedShapes, null, null); } @@ -1340,7 +1455,7 @@ private void stubGrantableUser(DbType dbType) { } private CreatePermissionCommand deniedCommand(List deniedColumns) { - return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, + return new CreatePermissionCommand(userId, true, false, false, false, null, null, null, null, null, deniedColumns, null, null, List.of(), null, null); } @@ -1408,7 +1523,7 @@ void grantGroupPermissionPersistsAndReturnsView() { .thenAnswer(inv -> inv.getArgument(0)); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, true, false, false, null, List.of("public"), null, null, null, null, null, List.of(), null); + groupId, true, true, false, false, null, null, List.of("public"), null, null, null, null, null, List.of(), null); var view = service.grantGroupPermission(datasourceId, orgId, adminId, command); assertThat(view.groupId()).isEqualTo(groupId); @@ -1439,7 +1554,7 @@ void grantGroupPermissionPersistsNormalizedDeniedSchemasAndTables() { var view = service.grantGroupPermission(datasourceId, orgId, adminId, new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, + groupId, true, false, false, false, null, null, null, null, null, null, List.of("Audit"), List.of("CRM.Salary", " "), List.of(), null)); assertThat(saved.getValue().getDeniedSchemas()).containsExactly("audit"); @@ -1464,7 +1579,7 @@ void grantGroupPermissionPersistsDeniedShapesAndRefusesThemOnAnEngineManagedData com.bablsoft.accessflow.core.internal.persistence.entity.DatasourceGroupPermissionEntity.class); when(groupPermissionRepository.save(saved.capture())).thenAnswer(inv -> inv.getArgument(0)); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, null, null, + groupId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(QueryShape.HAVING, QueryShape.GROUP_BY), null); var view = service.grantGroupPermission(datasourceId, orgId, adminId, command); @@ -1496,7 +1611,7 @@ void grantGroupPermissionStoresNoDeniedSchemasOrTablesWhenAbsent() { var view = service.grantGroupPermission(datasourceId, orgId, adminId, new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, + groupId, true, false, false, false, null, null, null, null, null, null, List.of(), null, List.of(), null)); assertThat(saved.getValue().getDeniedSchemas()).isNull(); @@ -1516,7 +1631,7 @@ void grantGroupPermissionRejectsDuplicate() { .thenReturn(true); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, null, null, List.of(), null); + groupId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null); assertThatThrownBy(() -> service.grantGroupPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(com.bablsoft.accessflow.core.api.DatasourceGroupPermissionAlreadyExistsException.class); } @@ -1530,7 +1645,7 @@ void grantGroupPermissionRejectsUnknownGroup() { .thenThrow(new com.bablsoft.accessflow.core.api.UserGroupNotFoundException(groupId)); var command = new com.bablsoft.accessflow.core.api.CreateDatasourceGroupPermissionCommand( - groupId, true, false, false, false, null, null, null, null, null, null, null, List.of(), null); + groupId, true, false, false, false, null, null, null, null, null, null, null, null, List.of(), null); assertThatThrownBy(() -> service.grantGroupPermission(datasourceId, orgId, adminId, command)) .isInstanceOf(com.bablsoft.accessflow.core.api.UserGroupNotFoundException.class); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java new file mode 100644 index 000000000..d801f3b04 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultBytesScannedCapResolutionServiceTest.java @@ -0,0 +1,126 @@ +package com.bablsoft.accessflow.core.internal; + +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionLookupService; +import com.bablsoft.accessflow.core.api.DatasourceUserPermissionView; +import com.bablsoft.accessflow.core.internal.persistence.entity.DatasourceEntity; +import com.bablsoft.accessflow.core.internal.persistence.repo.DatasourceRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DefaultBytesScannedCapResolutionServiceTest { + + @Mock DatasourceRepository datasourceRepository; + @Mock DatasourceUserPermissionLookupService permissionLookupService; + @InjectMocks DefaultBytesScannedCapResolutionService service; + + private final UUID datasourceId = UUID.randomUUID(); + private final UUID userId = UUID.randomUUID(); + + @Test + void noCapAnywhereResolvesToNothing() { + givenDatasource(null, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(null); + + assertThat(service.resolve(datasourceId, userId)).isEmpty(); + } + + @Test + void anUnknownDatasourceResolvesToNothing() { + when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.empty()); + + assertThat(service.resolve(datasourceId, userId)).isEmpty(); + verifyNoInteractions(permissionLookupService); + } + + @Test + void theDatasourceCapAppliesWhenNoGrantSetsOne() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REJECT); + givenGrant(null); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(1_000L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.DATASOURCE); + assertThat(cap.missingEstimate()).isEqualTo(BytesCapMissingEstimateAction.REJECT); + } + + @Test + void aTighterGrantWins() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(400L); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(400L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.GRANT); + } + + @Test + void aLooserGrantNeverWidensTheDatasourceCap() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(5_000L); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(1_000L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.DATASOURCE); + } + + @Test + void aTieIsAttributedToTheDatasource() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenGrant(1_000L); + + assertThat(service.resolve(datasourceId, userId).orElseThrow().source()) + .isEqualTo(BytesScannedCapSource.DATASOURCE); + } + + @Test + void aGrantCapAppliesOnAnUncappedDatasourceWithItsMissingEstimatePolicy() { + givenDatasource(null, BytesCapMissingEstimateAction.REJECT); + givenGrant(300L); + + var cap = service.resolve(datasourceId, userId).orElseThrow(); + + assertThat(cap.limit()).isEqualTo(300L); + assertThat(cap.source()).isEqualTo(BytesScannedCapSource.GRANT); + assertThat(cap.missingEstimate()).isEqualTo(BytesCapMissingEstimateAction.REJECT); + } + + @Test + void anAnonymousLookupOnlyReadsTheDatasource() { + givenDatasource(1_000L, BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + assertThat(service.resolve(datasourceId, null)).isPresent(); + verifyNoInteractions(permissionLookupService); + } + + private void givenDatasource(Long cap, BytesCapMissingEstimateAction missing) { + var entity = new DatasourceEntity(); + entity.setId(datasourceId); + entity.setMaxBytesScannedPerQuery(cap); + entity.setBytesCapMissingEstimate(missing); + when(datasourceRepository.findById(datasourceId)).thenReturn(Optional.of(entity)); + } + + private void givenGrant(Long override) { + var view = new DatasourceUserPermissionView(UUID.randomUUID(), userId, datasourceId, true, + false, false, false, List.of(), List.of(), List.of(), List.of(), List.of(), + List.of(), List.of(), null, override, null); + when(permissionLookupService.findFor(userId, datasourceId)).thenReturn(Optional.of(view)); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java index 4954d9034..142c8cee5 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultDatasourceUserPermissionLookupServiceTest.java @@ -544,6 +544,51 @@ void findDirectForAndContributionsCarryTheRowLimitOverride() { .isEqualTo(75); } + @Test + void findForTakesTheSmallestBytesScannedCapAcrossGrants() { + var userId = UUID.randomUUID(); + var datasourceId = UUID.randomUUID(); + var groupId = UUID.randomUUID(); + var direct = newPermission(UUID.randomUUID(), userId, datasourceId); + direct.setCanRead(true); + direct.setBytesScannedLimitOverride(5_000L); + var group = newGroupPermission(groupId, datasourceId); + group.setCanRead(true); + group.setBytesScannedLimitOverride(2_000L); + when(permissionRepository.findByUser_IdAndDatasource_Id(userId, datasourceId)) + .thenReturn(Optional.of(direct)); + when(membershipRepository.findGroupIdsForUser(userId)).thenReturn(List.of(groupId)); + when(groupPermissionRepository.findAllByGroup_IdIn(List.of(groupId))) + .thenReturn(List.of(group)); + + assertThat(service.findFor(userId, datasourceId).orElseThrow().bytesScannedLimitOverride()) + .isEqualTo(2_000L); + assertThat(service.findDirectFor(userId, datasourceId).orElseThrow() + .bytesScannedLimitOverride()).isEqualTo(5_000L); + assertThat(service.findContributions(userId, datasourceId)) + .extracting(c -> c.bytesScannedLimitOverride()).containsExactly(5_000L, 2_000L); + } + + @Test + void findForBytesScannedCapIsNullWhenNoGrantSetsOneAndOneSetOneWins() { + var userId = UUID.randomUUID(); + var datasourceId = UUID.randomUUID(); + var groupId = UUID.randomUUID(); + var direct = newPermission(UUID.randomUUID(), userId, datasourceId); + direct.setCanRead(true); + var group = newGroupPermission(groupId, datasourceId); + group.setCanRead(true); + group.setBytesScannedLimitOverride(700L); + when(permissionRepository.findByUser_IdAndDatasource_Id(userId, datasourceId)) + .thenReturn(Optional.of(direct)); + when(membershipRepository.findGroupIdsForUser(userId)).thenReturn(List.of(groupId)); + when(groupPermissionRepository.findAllByGroup_IdIn(List.of(groupId))) + .thenReturn(List.of(group)); + + assertThat(service.findFor(userId, datasourceId).orElseThrow().bytesScannedLimitOverride()) + .isEqualTo(700L); + } + private DatasourceUserPermissionView mergeRowLimits( Integer directLimit, Integer groupLimit) { var userId = UUID.randomUUID(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java index 7ef7e6718..460004424 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryEstimateServiceTest.java @@ -33,7 +33,7 @@ class DefaultQueryEstimateServiceTest { private static PersistQueryEstimateCommand command() { return new PersistQueryEstimateCommand("postgresql", QueryType.DELETE, true, 120L, 90L, - "Seq Scan", 44.5, "{\"operation\":\"Seq Scan\"}", "[raw]", null, false, null, 12); + "Seq Scan", 44.5, null, "{\"operation\":\"Seq Scan\"}", "[raw]", null, false, null, 12); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java index 326c72d26..e0e1cdf37 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestLookupServiceTest.java @@ -47,6 +47,7 @@ class DefaultQueryRequestLookupServiceTest { @Mock AiAnalysisRepository aiAnalysisRepository; @Mock ReviewDecisionRepository reviewDecisionRepository; @Mock ApprovalPredictionRepository approvalPredictionRepository; + @Mock com.bablsoft.accessflow.core.internal.persistence.repo.QueryEstimateRepository queryEstimateRepository; @InjectMocks DefaultQueryRequestLookupService service; @Test @@ -333,6 +334,51 @@ void findDetailByIdCopiesFailureFlagAndReason() { assertThat(detail.aiAnalysis().errorMessage()).isEqualTo("provider unavailable"); } + @Test + void findDetailByIdCarriesTheBytesScannedCapAndTheBytesEstimate() { + var orgId = UUID.randomUUID(); + var queryId = UUID.randomUUID(); + var entity = entityWith(queryId, UUID.randomUUID(), orgId, UUID.randomUUID(), + "alice@example.com", QueryStatus.REJECTED); + entity.setBytesScannedCap(1_000L); + entity.setBytesScannedCapSource(com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE); + entity.setBytesScannedCapOutcome(com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + var estimateId = UUID.randomUUID(); + entity.setQueryEstimateId(estimateId); + var estimate = new com.bablsoft.accessflow.core.internal.persistence.entity.QueryEstimateEntity(); + estimate.setId(estimateId); + estimate.setQueryRequest(entity); + estimate.setSupported(true); + estimate.setEstimatedBytesScanned(9_000L); + when(queryEstimateRepository.findById(estimateId)).thenReturn(Optional.of(estimate)); + when(queryRequestRepository.findById(queryId)).thenReturn(Optional.of(entity)); + when(reviewDecisionRepository.findAllByQueryRequest_IdOrderByDecidedAtAsc(queryId)) + .thenReturn(List.of()); + + var detail = service.findDetailById(queryId, orgId).orElseThrow(); + + assertThat(detail.costEstimate().estimatedBytesScanned()).isEqualTo(9_000L); + assertThat(detail.bytesScannedCap().limit()).isEqualTo(1_000L); + assertThat(detail.bytesScannedCap().source()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE); + assertThat(detail.bytesScannedCap().outcome()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + } + + @Test + void findDetailByIdLeavesTheBytesScannedCapNullWhenNoneApplied() { + var orgId = UUID.randomUUID(); + var queryId = UUID.randomUUID(); + var entity = entityWith(queryId, UUID.randomUUID(), orgId, UUID.randomUUID(), + "alice@example.com", QueryStatus.PENDING_AI); + entity.setBytesScannedCap(1_000L); + when(queryRequestRepository.findById(queryId)).thenReturn(Optional.of(entity)); + when(reviewDecisionRepository.findAllByQueryRequest_IdOrderByDecidedAtAsc(queryId)) + .thenReturn(List.of()); + + assertThat(service.findDetailById(queryId, orgId).orElseThrow().bytesScannedCap()).isNull(); + } + @Test void findDetailByIdLeavesAiAnalysisNullWhenNoAnalysisLinked() { var orgId = UUID.randomUUID(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java index f35c1d26e..7df5a0565 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/core/internal/DefaultQueryRequestStateServiceTest.java @@ -131,6 +131,25 @@ void approveByAccessGrantThrowsWhenNotPendingAi() { verify(queryRequestRepository, never()).save(any()); } + @Test + void recordBytesScannedCapStampsTheCapWithoutTransitioning() { + query.setStatus(QueryStatus.PENDING_AI); + when(queryRequestRepository.findByIdForUpdate(queryId)).thenReturn(Optional.of(query)); + + service.recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + + assertThat(query.getStatus()).isEqualTo(QueryStatus.PENDING_AI); + assertThat(query.getBytesScannedCap()).isEqualTo(1_000L); + assertThat(query.getBytesScannedCapSource()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT); + assertThat(query.getBytesScannedCapOutcome()) + .isEqualTo(com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + verify(queryRequestRepository).save(query); + verify(eventPublisher, never()).publishEvent(any()); + } + @Test void recordApprovalAndAdvancePromotesToApprovedAtLastStage() { query.setStatus(QueryStatus.PENDING_REVIEW); diff --git a/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java index 17e4934b4..0c8bac945 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/proxy/internal/DefaultQueryCostEstimateServiceTest.java @@ -84,7 +84,7 @@ private QueryRequestSnapshot snapshot(QueryType type, boolean transactional) { private QueryEstimateSnapshot persistedSnapshot() { return new QueryEstimateSnapshot(estimateId, queryRequestId, "postgresql", - QueryType.DELETE, true, 100L, 90L, "Seq Scan", 12.5, null, "raw", null, + QueryType.DELETE, true, 100L, 90L, "Seq Scan", 12.5, null, null, "raw", null, false, null, 3, Instant.now()); } @@ -165,6 +165,56 @@ void supportedDryRunWithAffectedCountPersistsFullRow() { .isEqualTo(Duration.ofSeconds(5)); } + @Test + void aWarehouseBytesEstimateIsPersistedWithTheRow() { + when(lookupService.findByQueryRequestId(queryRequestId)) + .thenReturn(Optional.empty()) + .thenReturn(Optional.of(persistedSnapshot())); + when(queryRequestLookupService.findById(queryRequestId)) + .thenReturn(Optional.of(snapshot(QueryType.SELECT, false))); + when(rowSecurityResolutionService.resolveApplicable(any(), any(), any())) + .thenReturn(List.of()); + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.of( + "bigquery", QueryType.SELECT, null, null, null, Set.of(), Duration.ofMillis(4)) + .withEstimatedBytesScanned(3_000_000_000L)); + when(persistenceService.persist(eq(queryRequestId), any())).thenReturn(estimateId); + + service.estimateSubmittedQuery(queryRequestId); + + var captor = ArgumentCaptor.forClass(PersistQueryEstimateCommand.class); + verify(persistenceService).persist(eq(queryRequestId), captor.capture()); + assertThat(captor.getValue().estimatedBytesScanned()).isEqualTo(3_000_000_000L); + } + + @Test + void estimateBytesScannedDryRunsUnderTheEstimateTimeoutAndPersistsNothing() { + var request = new QueryExecutionRequest(datasourceId, "SELECT 1", QueryType.SELECT, 50, + null, List.of(), List.of(), List.of(), false, null, List.of()); + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.of("bigquery", + QueryType.SELECT, null, null, null, Set.of(), Duration.ofMillis(3)) + .withEstimatedBytesScanned(42L)); + + assertThat(service.estimateBytesScanned(request)).contains(42L); + + var captor = ArgumentCaptor.forClass(QueryExecutionRequest.class); + verify(queryExecutor).dryRun(captor.capture()); + assertThat(captor.getValue().statementTimeoutOverride()).isEqualTo(Duration.ofSeconds(5)); + assertThat(captor.getValue().maxRowsOverride()).isEqualTo(50); + verifyNoInteractions(persistenceService, eventPublisher); + } + + @Test + void estimateBytesScannedIsEmptyWhenUnsupportedOrFailing() { + var request = new QueryExecutionRequest(datasourceId, "SELECT 1", QueryType.SELECT, null, + null, List.of(), List.of(), List.of(), false, null, List.of()); + when(queryExecutor.dryRun(any())) + .thenReturn(QueryDryRunResult.unsupported("redis")) + .thenThrow(new IllegalStateException("down")); + + assertThat(service.estimateBytesScanned(request)).isEmpty(); + assertThat(service.estimateBytesScanned(request)).isEmpty(); + } + @Test void writePlanDescendsIntoAccessNodeForScanTypeAndEstimate() { when(lookupService.findByQueryRequestId(queryRequestId)) diff --git a/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java index 7d60404be..d3a70b70b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/requestgroups/internal/GroupExecutionServiceTest.java @@ -61,6 +61,14 @@ class GroupExecutionServiceTest { private AuditLogService auditLogService; @Mock private org.springframework.context.ApplicationEventPublisher eventPublisher; + @Mock + private com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; + @Mock + private org.springframework.context.MessageSource messageSource; + @Mock + private com.bablsoft.accessflow.proxy.api.QueryCostEstimateService queryCostEstimateService; + @Mock + private com.bablsoft.accessflow.requestgroups.internal.persistence.repo.GroupReviewDecisionRepository decisionRepository; @InjectMocks private GroupExecutionService service; @@ -148,6 +156,121 @@ void ignoresGroupNotApproved() { verify(stateService, org.mockito.Mockito.never()).apply(any(), any()); } + private RequestGroupItemEntity queryItem() { + var item = new RequestGroupItemEntity(); + item.setId(UUID.randomUUID()); + item.setGroupId(group.getId()); + item.setSequenceOrder(0); + item.setTargetKind(RequestGroupTargetKind.QUERY); + item.setDatasourceId(UUID.randomUUID()); + item.setSqlText("SELECT 1"); + item.setQueryType(com.bablsoft.accessflow.core.api.QueryType.SELECT); + when(itemRepository.findByGroupIdOrderBySequenceOrderAsc(group.getId())) + .thenReturn(new ArrayList<>(List.of(item))); + when(permissionLookupService.findFor(any(), any())).thenReturn(java.util.Optional.empty()); + when(maskingPolicyResolutionService.resolveApplicable(any(), any(), any())).thenReturn(List.of()); + when(rowSecurityResolutionService.resolveApplicable(any(), any(), any())).thenReturn(List.of()); + when(datasourceLookupService.findById(any())).thenReturn(java.util.Optional.empty()); + when(queryParser.parse(any(), any())) + .thenReturn(new com.bablsoft.accessflow.core.api.SqlParseResult( + com.bablsoft.accessflow.core.api.QueryType.SELECT, "SELECT 1")); + return item; + } + + private void givenBytesCap(RequestGroupItemEntity item, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + when(bytesScannedCapResolutionService.resolve(item.getDatasourceId(), group.getSubmittedBy())) + .thenReturn(java.util.Optional.of(new com.bablsoft.accessflow.core.api.AppliedBytesCap( + 1_000L, com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + missing))); + } + + @Test + void aQueryMemberOverTheBytesCapFailsWithoutRunning() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.of(5_000L)); + when(messageSource.getMessage(org.mockito.ArgumentMatchers.eq("error.bytes_cap.exceeded"), + any(), any())).thenReturn("over the cap"); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.FAILED); + assertThat(item.getErrorMessage()).isEqualTo("over the cap"); + verify(queryExecutor, org.mockito.Mockito.never()).execute(any()); + var audit = org.mockito.ArgumentCaptor.forClass( + com.bablsoft.accessflow.audit.api.AuditEntry.class); + verify(auditLogService, org.mockito.Mockito.atLeastOnce()).record(audit.capture()); + assertThat(audit.getAllValues()).anySatisfy(entry -> { + assertThat(entry.action()) + .isEqualTo(com.bablsoft.accessflow.audit.api.AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED); + assertThat(entry.metadata()).containsEntry("estimated_bytes", 5_000L) + .containsEntry("item_id", item.getId().toString()); + }); + } + + @Test + void aQueryMemberWithoutAnEstimateFailsUnderReject() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.empty()); + when(messageSource.getMessage(org.mockito.ArgumentMatchers.eq("error.bytes_cap.no_estimate"), + any(), any())).thenReturn("no estimate"); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.FAILED); + assertThat(item.getErrorMessage()).isEqualTo("no estimate"); + } + + @Test + void aQueryMemberWithoutAnEstimateRunsUnderRequireReviewOnceAPersonApproved() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.empty()); + when(decisionRepository.existsByRequestGroupIdAndDecision(group.getId(), + com.bablsoft.accessflow.core.api.DecisionType.APPROVED)).thenReturn(true); + when(queryExecutor.execute(any())).thenReturn( + new com.bablsoft.accessflow.core.api.SelectExecutionResult( + List.of(), List.of(), 1L, false, java.time.Duration.ofMillis(3))); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.EXECUTED); + } + + @Test + void aQueryMemberWithoutAnEstimateIsRefusedWhenNoPersonApprovedTheGroup() { + var item = queryItem(); + givenBytesCap(item, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryCostEstimateService.estimateBytesScanned(any())) + .thenReturn(java.util.Optional.empty()); + when(messageSource.getMessage( + org.mockito.ArgumentMatchers.eq("error.bytes_cap.no_estimate_unreviewed"), any(), + any())).thenReturn("never reviewed"); + + service.execute(group.getId(), null, "manual"); + + assertThat(item.getStatus()).isEqualTo(RequestGroupItemStatus.FAILED); + assertThat(item.getErrorMessage()).isEqualTo("never reviewed"); + verify(queryExecutor, org.mockito.Mockito.never()).execute(any()); + } + + @Test + void anUncappedQueryMemberIsNeverDryRun() { + queryItem(); + when(queryExecutor.execute(any())).thenReturn( + new com.bablsoft.accessflow.core.api.SelectExecutionResult( + List.of(), List.of(), 1L, false, java.time.Duration.ofMillis(3))); + + service.execute(group.getId(), null, "manual"); + + verify(queryCostEstimateService, org.mockito.Mockito.never()).estimateBytesScanned(any()); + } + @Test void runsQueryMemberThroughProxyExecutor() { var item = new RequestGroupItemEntity(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java index 0343cf896..c0e3389cf 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/DatasourceControllerIntegrationTest.java @@ -520,6 +520,101 @@ void updateDatasourceSetsKeepsAndClearsTheEnvironment() { assertThat(datasourceRepository.findById(ds.getId()).orElseThrow().getEnvironment()).isNull(); } + @Test + void updateDatasourceSetsKeepsAndClearsTheBytesScannedCapOnAWarehouse() { + var ds = saveDatasource(primaryOrg, "WH"); + ds.setDbType(com.bablsoft.accessflow.core.api.DbType.BIGQUERY); + datasourceRepository.save(ds); + + var set = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"max_bytes_scanned_per_query":1000000000000,"bytes_cap_missing_estimate":"REJECT"} + """) + .exchange(); + assertThat(set).hasStatus(200); + assertThat(set).bodyJson().extractingPath("$.max_bytes_scanned_per_query").asNumber() + .isEqualTo(1_000_000_000_000L); + assertThat(set).bodyJson().extractingPath("$.bytes_cap_missing_estimate").asString() + .isEqualTo("REJECT"); + + var keep = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"host":"other-host"} + """) + .exchange(); + assertThat(keep).bodyJson().extractingPath("$.max_bytes_scanned_per_query").asNumber() + .isEqualTo(1_000_000_000_000L); + + var clear = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"clear_max_bytes_scanned_per_query":true} + """) + .exchange(); + assertThat(clear).hasStatus(200); + assertThat(clear).bodyJson().doesNotHavePath("$.max_bytes_scanned_per_query"); + } + + @Test + void aBytesScannedCapIsRefusedOnAnEngineWithoutABytesEstimateAndBelowOne() { + var ds = saveDatasource(primaryOrg, "DS"); + + var unsupported = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"max_bytes_scanned_per_query":5} + """) + .exchange(); + assertThat(unsupported).hasStatus(422); + assertThat(unsupported).bodyJson().extractingPath("$.error").asString() + .isEqualTo("BYTES_SCANNED_CAP_NOT_SUPPORTED"); + + var zero = mvc.put().uri("/api/v1/datasources/" + ds.getId()) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"max_bytes_scanned_per_query":0} + """) + .exchange(); + assertThat(zero).hasStatus(400); + + var grant = mvc.post().uri("/api/v1/datasources/" + ds.getId() + "/permissions") + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"user_id":"%s","can_read":true,"bytes_scanned_limit_override":5} + """.formatted(analyst.getId())) + .exchange(); + assertThat(grant).hasStatus(422); + assertThat(permissionRepository.existsByUser_IdAndDatasource_Id(analyst.getId(), + ds.getId())).isFalse(); + } + + @Test + void grantPermissionRoundTripsTheBytesScannedOverrideOnAWarehouse() { + var ds = saveDatasource(primaryOrg, "WH"); + ds.setDbType(com.bablsoft.accessflow.core.api.DbType.SNOWFLAKE); + datasourceRepository.save(ds); + + var result = mvc.post().uri("/api/v1/datasources/" + ds.getId() + "/permissions") + .header(HttpHeaders.AUTHORIZATION, "Bearer " + adminToken) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + {"user_id":"%s","can_read":true,"bytes_scanned_limit_override":500000000000} + """.formatted(analyst.getId())) + .exchange(); + + assertThat(result).hasStatus(201); + assertThat(result).bodyJson().extractingPath("$.bytes_scanned_limit_override").asNumber() + .isEqualTo(500_000_000_000L); + } + @Test void updateDatasourcePasswordReencryptsIt() { var ds = saveDatasource(primaryOrg, "DS"); diff --git a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java index 156aa50e5..0f6fd1cf0 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/security/internal/web/GlobalExceptionHandlerTest.java @@ -303,6 +303,18 @@ void deniedShapesNotSupportedReturns422() { .containsEntry("dbType", "REDIS"); } + @Test + void bytesScannedCapNotSupportedReturns422() { + var pd = handler.handleBytesScannedCapNotSupported( + new com.bablsoft.accessflow.core.api.BytesScannedCapNotSupportedException( + com.bablsoft.accessflow.core.api.DbType.POSTGRESQL)); + + assertThat(pd.getStatus()).isEqualTo(422); + assertThat(pd.getProperties()) + .containsEntry("error", "BYTES_SCANNED_CAP_NOT_SUPPORTED") + .containsEntry("dbType", "POSTGRESQL"); + } + @Test void invalidSqlReturns422() { var pd = handler.handleInvalidSql(new InvalidSqlException("nope")); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java index 6b32b3085..f0b917889 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/api/ConditionNodeTest.java @@ -169,4 +169,24 @@ void ciCdOriginExposesExpected() { assertThat(new ConditionNode.CiCdOrigin(true).expected()).isTrue(); assertThat(new ConditionNode.CiCdOrigin(false).expected()).isFalse(); } + + @Test + void estimatedBytesScannedRejectsANullOperatorAndANegativeValue() { + assertThatThrownBy(() -> new ConditionNode.EstimatedBytesScanned(null, 5)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, -1)) + .isInstanceOf(IllegalArgumentException.class); + var node = new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, 1_000_000L); + assertThat(node.operator()).isEqualTo(ComparisonOperator.GT); + assertThat(node.value()).isEqualTo(1_000_000L); + } + + @Test + void theShapeCompatibleContextConstructorLeavesTheBytesEstimateAbsent() { + var context = new ConditionContext(QueryType.SELECT, Set.of(), RiskLevel.LOW, 1, "ANALYST", + Set.of(), java.time.LocalDateTime.now(), false, false, false, null, null, false, + null, false, 10L, "Seq Scan", Set.of(QueryShape.JOIN), true); + assertThat(context.estimatedBytesScanned()).isNull(); + assertThat(context.shapesAnalyzed()).isTrue(); + } } diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java new file mode 100644 index 000000000..ed8f84ba2 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesCapCheckTest.java @@ -0,0 +1,48 @@ +package com.bablsoft.accessflow.workflow.internal; + +import com.bablsoft.accessflow.core.api.AppliedBytesCap; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class BytesCapCheckTest { + + private final AppliedBytesCap cap = new AppliedBytesCap(100, BytesScannedCapSource.DATASOURCE, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + @Test + void ofComparesTheEstimate() { + var over = BytesCapCheck.of(cap, 101L); + assertThat(over.outcome()).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + assertThat(over.rejects()).isTrue(); + assertThat(over.forcesReview()).isFalse(); + assertThat(over.limit()).isEqualTo(100); + assertThat(over.estimatedBytes()).isEqualTo(101L); + } + + @Test + void aMissingEstimateUnderRequireReviewForcesReviewWithoutRejecting() { + var check = BytesCapCheck.of(cap, null); + assertThat(check.forcesReview()).isTrue(); + assertThat(check.rejects()).isFalse(); + } + + @Test + void anUnevaluatedCapNeitherRejectsNorForcesReview() { + var check = BytesCapCheck.unevaluated(cap); + assertThat(check.outcome()).isNull(); + assertThat(check.rejects()).isFalse(); + assertThat(check.forcesReview()).isFalse(); + assertThat(check.source()).isEqualTo(BytesScannedCapSource.DATASOURCE); + } + + @Test + void aSourceIsRequired() { + assertThatThrownBy(() -> new BytesCapCheck(1, null, null, null)) + .isInstanceOf(NullPointerException.class); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java new file mode 100644 index 000000000..165db3237 --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/BytesScannedCapEnforcementIntegrationTest.java @@ -0,0 +1,355 @@ +package com.bablsoft.accessflow.workflow.internal; + +import com.bablsoft.accessflow.TestcontainersConfig; +import com.bablsoft.accessflow.core.api.AuthProviderType; +import com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; +import com.bablsoft.accessflow.core.api.CredentialEncryptionService; +import com.bablsoft.accessflow.core.api.DbType; +import com.bablsoft.accessflow.core.api.QueryDryRunResult; +import com.bablsoft.accessflow.core.api.QueryStatus; +import com.bablsoft.accessflow.core.api.QueryType; +import com.bablsoft.accessflow.core.api.RiskLevel; +import com.bablsoft.accessflow.core.api.SelectExecutionResult; +import com.bablsoft.accessflow.core.api.SslMode; +import com.bablsoft.accessflow.core.api.UserRoleType; +import com.bablsoft.accessflow.core.events.AiAnalysisCompletedEvent; +import com.bablsoft.accessflow.core.events.AiAnalysisSkippedEvent; +import com.bablsoft.accessflow.core.internal.persistence.entity.DatasourceEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.OrganizationEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.QueryRequestEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.ReviewPlanApproverEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.ReviewPlanEntity; +import com.bablsoft.accessflow.core.internal.persistence.entity.UserEntity; +import com.bablsoft.accessflow.core.internal.persistence.repo.DatasourceRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.OrganizationRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.QueryEstimateRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.QueryRequestRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.ReviewPlanApproverRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.ReviewPlanRepository; +import com.bablsoft.accessflow.core.internal.persistence.repo.UserRepository; +import com.bablsoft.accessflow.core.api.SqlParseResult; +import com.bablsoft.accessflow.proxy.api.QueryExecutor; +import com.bablsoft.accessflow.proxy.api.QueryParser; +import com.bablsoft.accessflow.workflow.api.ComparisonOperator; +import com.bablsoft.accessflow.workflow.api.ConditionNode; +import com.bablsoft.accessflow.workflow.api.QueryLifecycleService; +import com.bablsoft.accessflow.workflow.api.QueryLifecycleService.ExecuteQueryCommand; +import com.bablsoft.accessflow.workflow.api.RoutingAction; +import com.bablsoft.accessflow.workflow.internal.persistence.entity.RoutingPolicyEntity; +import com.bablsoft.accessflow.workflow.internal.persistence.repo.RoutingDecisionRepository; +import com.bablsoft.accessflow.workflow.internal.persistence.repo.RoutingPolicyRepository; +import com.bablsoft.accessflow.workflow.internal.routing.RoutingConditionCodec; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.testcontainers.context.ImportTestcontainers; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * #941 end to end: the warehouse's pre-flight bytes estimate is persisted with the query's + * estimate, a bytes-scanned cap refuses the query when it leaves {@code PENDING_AI}, a missing + * estimate under {@code REQUIRE_REVIEW} holds an automatic approval for a person, the cap is + * re-checked just before execution, and an {@code estimated_bytes_scanned} routing policy sees the + * estimate on the AI-skipped path, where nothing else waits for it. + * + *

No warehouse is contacted: {@link QueryExecutor} is mocked, so its {@code dryRun} supplies the + * estimate the BigQuery engine would report. + */ +@SpringBootTest +@ImportTestcontainers(TestcontainersConfig.class) +class BytesScannedCapEnforcementIntegrationTest { + + private static final String SQL = "SELECT * FROM events"; + private static final long ONE_TB = 1_000_000_000_000L; + + @MockitoBean QueryExecutor queryExecutor; + // No BigQuery engine plugin is loaded in tests; the parser is the plugin's to supply. + @MockitoBean QueryParser queryParser; + + @Autowired ApplicationEventPublisher eventPublisher; + @Autowired PlatformTransactionManager transactionManager; + @Autowired QueryLifecycleService queryLifecycleService; + @Autowired OrganizationRepository organizationRepository; + @Autowired UserRepository userRepository; + @Autowired DatasourceRepository datasourceRepository; + @Autowired QueryRequestRepository queryRequestRepository; + @Autowired QueryEstimateRepository queryEstimateRepository; + @Autowired ReviewPlanRepository reviewPlanRepository; + @Autowired ReviewPlanApproverRepository reviewPlanApproverRepository; + @Autowired RoutingPolicyRepository routingPolicyRepository; + @Autowired RoutingDecisionRepository routingDecisionRepository; + @Autowired RoutingConditionCodec routingConditionCodec; + @Autowired CredentialEncryptionService encryptionService; + @Autowired JdbcTemplate jdbcTemplate; + + private OrganizationEntity organization; + private UserEntity submitter; + + @BeforeEach + void setUp() { + cleanup(); + organization = new OrganizationEntity(); + organization.setId(UUID.randomUUID()); + organization.setName("Primary"); + organization.setSlug("primary-" + UUID.randomUUID()); + organizationRepository.save(organization); + submitter = persistUser(); + when(queryParser.parse(any(), any())).thenReturn(new SqlParseResult(QueryType.SELECT, SQL)); + } + + @AfterEach + void cleanup() { + if (organization != null) { + jdbcTemplate.update("DELETE FROM audit_log WHERE organization_id = ?", organization.getId()); + } + routingDecisionRepository.deleteAll(); + routingPolicyRepository.deleteAll(); + jdbcTemplate.update("UPDATE query_requests SET query_estimate_id = NULL"); + queryEstimateRepository.deleteAll(); + queryRequestRepository.deleteAll(); + datasourceRepository.deleteAll(); + reviewPlanApproverRepository.deleteAll(); + reviewPlanRepository.deleteAll(); + userRepository.deleteAll(); + organizationRepository.deleteAll(); + } + + @Test + void anEstimateOverTheCapIsRejectedWhenTheQueryLeavesPendingAi() { + givenBytesEstimate(2 * ONE_TB); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisCompletedEvent(query.getId(), null, RiskLevel.LOW, 5)); + + awaitStatus(query.getId(), QueryStatus.REJECTED); + var row = queryRequestRepository.findById(query.getId()).orElseThrow(); + assertThat(row.getBytesScannedCap()).isEqualTo(ONE_TB); + assertThat(row.getBytesScannedCapSource()).isEqualTo(BytesScannedCapSource.DATASOURCE); + assertThat(row.getBytesScannedCapOutcome()).isEqualTo(BytesScannedCapOutcome.EXCEEDED); + assertThat(queryEstimateRepository.findByQueryRequestId(query.getId()).orElseThrow() + .getEstimatedBytesScanned()).isEqualTo(2 * ONE_TB); + assertThat(capAuditMetadata(query.getId())) + .contains("\"stage\": \"decision\"") + .contains("\"outcome\": \"EXCEEDED\""); + } + + @Test + void aMissingEstimateUnderRequireReviewHoldsAnAutoApprovalForAPerson() { + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.unsupported("bigquery")); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisSkippedEvent(query.getId(), "ai_analysis_enabled=false")); + + awaitStatus(query.getId(), QueryStatus.PENDING_REVIEW); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow() + .getBytesScannedCapOutcome()).isEqualTo(BytesScannedCapOutcome.NO_ESTIMATE_REVIEW); + assertThat(capAuditMetadata(query.getId())).contains("NO_ESTIMATE_REVIEW"); + } + + @Test + void aCapLoweredAfterApprovalFailsTheExecutionBeforeTheWarehouseRunsIt() { + givenBytesEstimate(500_000_000_000L); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var query = persistPendingAiQuery(datasource); + publish(new AiAnalysisCompletedEvent(query.getId(), null, RiskLevel.LOW, 5)); + awaitStatus(query.getId(), QueryStatus.APPROVED); + + datasource.setMaxBytesScannedPerQuery(100_000_000_000L); + datasourceRepository.save(datasource); + var outcome = queryLifecycleService.execute(new ExecuteQueryCommand(query.getId(), + submitter.getId(), organization.getId(), false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.FAILED); + verify(queryParser, never()).parse(any(), any()); + verify(queryExecutor, never()).execute(any()); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow().getErrorMessage()) + .contains("500 GB").contains("100 GB"); + assertThat(capAuditMetadata(query.getId())).contains("\"stage\": \"execution\""); + } + + @Test + void anEstimateWithinTheCapExecutes() { + givenBytesEstimate(10L); + when(queryExecutor.execute(any())).thenReturn(new SelectExecutionResult(List.of(), + List.of(), 0L, false, Duration.ofMillis(3))); + var datasource = persistDatasource(persistPlan(false), ONE_TB, + BytesCapMissingEstimateAction.REJECT); + var query = persistPendingAiQuery(datasource); + publish(new AiAnalysisCompletedEvent(query.getId(), null, RiskLevel.LOW, 5)); + awaitStatus(query.getId(), QueryStatus.APPROVED); + + var outcome = queryLifecycleService.execute(new ExecuteQueryCommand(query.getId(), + submitter.getId(), organization.getId(), false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.EXECUTED); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow() + .getBytesScannedCapOutcome()).isEqualTo(BytesScannedCapOutcome.WITHIN); + assertThat(jdbcTemplate.queryForObject("SELECT count(*) FROM audit_log WHERE resource_id = ? " + + "AND action = 'QUERY_BYTES_SCANNED_CAP_ENFORCED'", Integer.class, query.getId())) + .isZero(); + } + + @Test + void anEstimatedBytesPolicyEscalatesOnTheSkippedPathWithoutAnyCap() { + givenBytesEstimate(2 * ONE_TB); + var datasource = persistDatasource(persistPlan(false), null, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + var policy = persistPolicy(new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, + ONE_TB)); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisSkippedEvent(query.getId(), "ai_analysis_enabled=false")); + + awaitStatus(query.getId(), QueryStatus.PENDING_REVIEW); + assertThat(routingDecisionRepository.findByQueryRequestId(query.getId()).orElseThrow() + .getMatchedPolicyId()).isEqualTo(policy.getId()); + assertThat(queryRequestRepository.findById(query.getId()).orElseThrow() + .getBytesScannedCap()).isNull(); + } + + @Test + void anEstimatedBytesPolicyDoesNotFireWithoutAnEstimate() { + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.unsupported("bigquery")); + var datasource = persistDatasource(persistPlan(false), null, + BytesCapMissingEstimateAction.REQUIRE_REVIEW); + persistPolicy(new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GTE, 0)); + var query = persistPendingAiQuery(datasource); + + publish(new AiAnalysisSkippedEvent(query.getId(), "ai_analysis_enabled=false")); + + awaitStatus(query.getId(), QueryStatus.APPROVED); + assertThat(routingDecisionRepository.findByQueryRequestId(query.getId())).isEmpty(); + } + + // ── Fixtures ────────────────────────────────────────────────────────────── + + private void givenBytesEstimate(long bytes) { + when(queryExecutor.dryRun(any())).thenReturn(QueryDryRunResult.of("bigquery", + QueryType.SELECT, null, null, null, Set.of(), Duration.ofMillis(2)) + .withEstimatedBytesScanned(bytes)); + } + + private void publish(Object event) { + new TransactionTemplate(transactionManager) + .executeWithoutResult(status -> eventPublisher.publishEvent(event)); + } + + private void awaitStatus(UUID queryId, QueryStatus expected) { + Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> + assertThat(queryRequestRepository.findById(queryId).orElseThrow().getStatus()) + .isEqualTo(expected)); + } + + private String capAuditMetadata(UUID queryId) { + var rows = jdbcTemplate.queryForList("SELECT metadata::text FROM audit_log WHERE " + + "resource_id = ? AND action = 'QUERY_BYTES_SCANNED_CAP_ENFORCED'", String.class, + queryId); + assertThat(rows).hasSize(1); + return rows.get(0); + } + + private UserEntity persistUser() { + var user = new UserEntity(); + user.setId(UUID.randomUUID()); + user.setEmail("submitter-" + UUID.randomUUID() + "@example.com"); + user.setDisplayName("submitter"); + user.setPasswordHash("hash"); + user.setRole(UserRoleType.ANALYST); + user.setAuthProvider(AuthProviderType.LOCAL); + user.setActive(true); + user.setOrganization(organization); + return userRepository.save(user); + } + + private ReviewPlanEntity persistPlan(boolean requiresHumanApproval) { + var plan = new ReviewPlanEntity(); + plan.setId(UUID.randomUUID()); + plan.setOrganization(organization); + plan.setName("plan-" + UUID.randomUUID()); + plan.setRequiresAiReview(false); + plan.setRequiresHumanApproval(requiresHumanApproval); + plan.setMinApprovalsRequired(1); + plan.setApprovalTimeoutHours(24); + plan.setAutoApproveReads(false); + reviewPlanRepository.save(plan); + var rule = new ReviewPlanApproverEntity(); + rule.setId(UUID.randomUUID()); + rule.setReviewPlan(plan); + rule.setRole("REVIEWER"); + rule.setStage(1); + reviewPlanApproverRepository.save(rule); + return plan; + } + + private DatasourceEntity persistDatasource(ReviewPlanEntity plan, Long cap, + BytesCapMissingEstimateAction missing) { + var ds = new DatasourceEntity(); + ds.setId(UUID.randomUUID()); + ds.setOrganization(organization); + ds.setName("WH-" + UUID.randomUUID()); + ds.setDbType(DbType.BIGQUERY); + ds.setHost("bigquery.invalid"); + ds.setPort(443); + ds.setDatabaseName("analytics"); + ds.setUsername("svc"); + ds.setPasswordEncrypted(encryptionService.encrypt("{}")); + ds.setSslMode(SslMode.REQUIRE); + ds.setConnectionPoolSize(5); + ds.setMaxRowsPerQuery(1000); + ds.setReviewPlan(plan); + ds.setAiAnalysisEnabled(false); + ds.setActive(true); + ds.setMaxBytesScannedPerQuery(cap); + ds.setBytesCapMissingEstimate(missing); + return datasourceRepository.save(ds); + } + + private QueryRequestEntity persistPendingAiQuery(DatasourceEntity ds) { + var query = new QueryRequestEntity(); + query.setId(UUID.randomUUID()); + query.setDatasource(ds); + query.setSubmittedBy(submitter); + query.setSqlText(SQL); + query.setQueryType(QueryType.SELECT); + query.setStatus(QueryStatus.PENDING_AI); + return queryRequestRepository.save(query); + } + + private RoutingPolicyEntity persistPolicy(ConditionNode condition) { + var policy = new RoutingPolicyEntity(); + policy.setId(UUID.randomUUID()); + policy.setOrganizationId(organization.getId()); + policy.setName("policy-" + UUID.randomUUID()); + policy.setPriority(1); + policy.setEnabled(true); + policy.setAction(RoutingAction.ESCALATE); + policy.setRequiredApprovals(1); + policy.setConditionJson(routingConditionCodec.encode(condition)); + return routingPolicyRepository.save(policy); + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java index 0afae9775..99719f337 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultAccessSimulationServiceTest.java @@ -98,6 +98,7 @@ class DefaultAccessSimulationServiceTest { @Mock RowSecurityClassificationService rowSecurityClassificationService; @Mock MaskingPolicyResolutionService maskingPolicyResolutionService; @Mock BreakGlassEligibilityService breakGlassEligibilityService; + @Mock com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; private DefaultAccessSimulationService service; @@ -114,7 +115,7 @@ void buildService() { routingPolicyEngine, reviewPlanLookupService, reviewerEligibilityService, rowSecurityResolutionService, rowSecurityClassificationService, maskingPolicyResolutionService, - breakGlassEligibilityService); + breakGlassEligibilityService, bytesScannedCapResolutionService); service.setClock(clock); } @@ -135,7 +136,7 @@ void stubHappyPath() { when(permissionLookupService.findFor(userId, datasourceId)) .thenReturn(Optional.of(permission(true, false, false, List.of("public")))); when(queryDecisionEvaluator.evaluate(any(), any(), any(), org.mockito.ArgumentMatchers.anyInt(), - any(), any())).thenReturn(planDecision(QueryStatus.PENDING_REVIEW)); + any(), any(), any())).thenReturn(planDecision(QueryStatus.PENDING_REVIEW)); when(sqlReviewService.evaluate(eq(organizationId), eq(datasourceId), any())) .thenReturn(SqlReviewResult.clean()); when(routingPolicyEngine.enabledFor(organizationId, datasourceId)).thenReturn(List.of()); @@ -175,7 +176,7 @@ void aSimulationNeverAsksTheEvaluatorToApplyAnything() { service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); verify(queryDecisionEvaluator).evaluate(any(), eq(AiOutcome.COMPLETED), eq(RiskLevel.LOW), - eq(5), eq(List.of()), eq(clock)); + eq(5), eq(List.of()), org.mockito.ArgumentMatchers.isNull(), eq(clock)); verify(datasourceAdminService, never()).update(any(), any(), any()); } @@ -195,7 +196,8 @@ void theSimulatorHandsTheEvaluatorTheSameBlockingRuleIdsTheLivePathWouldRead() { service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); verify(queryDecisionEvaluator).evaluate(any(), eq(AiOutcome.COMPLETED), eq(RiskLevel.LOW), - eq(5), eq(List.of("cross_join", "select_star")), eq(clock)); + eq(5), eq(List.of("cross_join", "select_star")), + org.mockito.ArgumentMatchers.isNull(), eq(clock)); verify(sqlReviewService).evaluate(organizationId, datasourceId, "SELECT card_number FROM public.payments"); } @@ -208,7 +210,24 @@ void aNonRelationalDatasourceContributesNoBlockingRuleIds() { service.simulate(organizationId, input(AiOutcome.SKIPPED, null, null)); verify(queryDecisionEvaluator).evaluate(any(), eq(AiOutcome.SKIPPED), eq(null), eq(-1), - eq(List.of()), eq(clock)); + eq(List.of()), org.mockito.ArgumentMatchers.isNull(), eq(clock)); + } + + @Test + void aBytesCapIsHandedToTheEvaluatorUnevaluatedBecauseASimulationHasNoEstimate() { + when(bytesScannedCapResolutionService.resolve(datasourceId, userId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.AppliedBytesCap(500L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT))); + + service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); + + var passed = org.mockito.ArgumentCaptor.forClass(BytesCapCheck.class); + verify(queryDecisionEvaluator).evaluate(any(), any(), any(), + org.mockito.ArgumentMatchers.anyInt(), any(), passed.capture(), any()); + assertThat(passed.getValue().limit()).isEqualTo(500L); + assertThat(passed.getValue().outcome()).isNull(); + assertThat(passed.getValue().rejects()).isFalse(); } // ── Shape ───────────────────────────────────────────────────────────────── @@ -236,7 +255,7 @@ void everyStageIsStillReportedWhenTheRequestIsBlockedEarly() { assertThat(step(result.steps(), QueryDecisionStepKind.ROUTING_POLICIES).outcome()) .isEqualTo(StepOutcome.SKIP); verify(queryDecisionEvaluator, never()).evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), any()); + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any()); } @Test @@ -514,7 +533,7 @@ void theRoutingStepCarriesEveryPolicyNotJustTheWinner() { @Test void reviewersAreSkippedWhenTheRequestWouldNotReachReview() { when(queryDecisionEvaluator.evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), any())) + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any())) .thenReturn(planDecision(QueryStatus.APPROVED)); var result = service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); @@ -534,13 +553,13 @@ void theSimulatorEvaluatesInTheSameZoneTheLiveListenerDoes() { routingPolicyEngine, reviewPlanLookupService, reviewerEligibilityService, rowSecurityResolutionService, rowSecurityClassificationService, maskingPolicyResolutionService, - breakGlassEligibilityService); + breakGlassEligibilityService, bytesScannedCapResolutionService); fresh.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); var passed = org.mockito.ArgumentCaptor.forClass(Clock.class); verify(queryDecisionEvaluator).evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), passed.capture()); + org.mockito.ArgumentMatchers.anyInt(), any(), any(), passed.capture()); assertThat(passed.getValue().getZone()).isEqualTo(ZoneId.systemDefault()); } @@ -583,7 +602,7 @@ void theDecisivePolicyIsTheOneTheDecisionNamesNotASecondEvaluationsOwnFirstMatch when(routingPolicyEngine.evaluateAll(any(), any())) .thenReturn(List.of(alsoMatched, decided)); when(queryDecisionEvaluator.evaluate(any(), any(), any(), - org.mockito.ArgumentMatchers.anyInt(), any(), any())) + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any())) .thenReturn(routedDecision(decidedId)); var result = service.simulate(organizationId, input(AiOutcome.COMPLETED, RiskLevel.LOW, 5)); @@ -724,6 +743,7 @@ private AccessSimulationInput input(AiOutcome outcome, RiskLevel level, Integer private QueryDecision planDecision(QueryStatus status) { var steps = List.of( DecisionTraceStep.of(QueryDecisionStepKind.SQL_REVIEW, StepOutcome.NO_MATCH, "k"), + DecisionTraceStep.of(QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.DENY, "k")); @@ -743,6 +763,7 @@ private QueryDecision routedDecision(UUID policyId) { "P", com.bablsoft.accessflow.workflow.api.RoutingAction.ESCALATE, 1, "matched"); var steps = List.of( DecisionTraceStep.of(QueryDecisionStepKind.SQL_REVIEW, StepOutcome.NO_MATCH, "k"), + DecisionTraceStep.of(QueryDecisionStepKind.BYTES_SCANNED_CAP, StepOutcome.NO_MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.MATCH, "k"), DecisionTraceStep.of(QueryDecisionStepKind.GRANT_FAST_PATH, StepOutcome.SKIP, "k"), DecisionTraceStep.of(QueryDecisionStepKind.REVIEW_PLAN, StepOutcome.SKIP, "k")); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java index 8ffa5b9a9..19d52f703 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/DefaultQueryLifecycleServiceTest.java @@ -95,6 +95,8 @@ class DefaultQueryLifecycleServiceTest { @Mock AuditLogService auditLogService; @Mock MessageSource messageSource; @Mock ApplicationEventPublisher eventPublisher; + @Mock com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; + @Mock com.bablsoft.accessflow.proxy.api.QueryCostEstimateService queryCostEstimateService; DefaultQueryLifecycleService service; @@ -132,7 +134,9 @@ void setUp() { auditLogService, new ObjectMapper(), messageSource, - eventPublisher); + eventPublisher, + bytesScannedCapResolutionService, + queryCostEstimateService); when(queryParser.parse(anyString(), any())).thenAnswer(inv -> { String sql = inv.getArgument(0); return new SqlParseResult(QueryType.SELECT, sql); @@ -618,6 +622,108 @@ void executeThrowsNotFoundWhenQueryMissing() { // ── execute (failure) ───────────────────────────────────────────────────── + // ── execute: bytes-scanned cap (#941) ───────────────────────────────────── + + private void givenBytesCap(long limit, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + when(bytesScannedCapResolutionService.resolve(datasourceId, submitterId)) + .thenReturn(Optional.of(new com.bablsoft.accessflow.core.api.AppliedBytesCap(limit, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, missing))); + } + + private void givenBytesEstimate(Long bytes) { + when(queryCostEstimateService.estimateSubmittedQuery(queryId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), + queryId, "bigquery", QueryType.SELECT, true, null, null, null, null, + bytes, null, null, null, false, null, 5, java.time.Instant.now()))); + } + + @Test + void executeFailsBeforeTheExecutorWhenTheEstimateExceedsTheCap() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenBytesEstimate(9_000L); + when(messageSource.getMessage(eq("error.bytes_cap.exceeded"), any(), any())) + .thenReturn("estimate over the cap"); + + var outcome = service.execute(new ExecuteQueryCommand(queryId, submitterId, + organizationId, false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.FAILED); + verify(queryExecutor, never()).execute(any()); + var execCaptor = ArgumentCaptor.forClass(RecordExecutionCommand.class); + verify(queryRequestStateService).recordExecutionOutcome(execCaptor.capture()); + assertThat(execCaptor.getValue().errorMessage()).isEqualTo("estimate over the cap"); + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService, org.mockito.Mockito.times(2)).record(auditCaptor.capture()); + assertThat(auditCaptor.getAllValues()).extracting(AuditEntry::action).containsExactly( + AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED, AuditAction.QUERY_FAILED); + assertThat(auditCaptor.getAllValues().get(0).metadata()) + .containsEntry("stage", "execution") + .containsEntry("estimated_bytes", 9_000L) + .containsEntry("source", "GRANT"); + } + + @Test + void executeFailsAMissingEstimateUnderReject() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + when(queryCostEstimateService.estimateSubmittedQuery(queryId)) + .thenThrow(new IllegalStateException("warehouse down")); + when(messageSource.getMessage(eq("error.bytes_cap.no_estimate"), any(), any())) + .thenReturn("no estimate"); + + var outcome = service.executeBreakGlass(queryId, submitterId); + + assertThat(outcome.status()).isEqualTo(QueryStatus.FAILED); + verify(queryExecutor, never()).execute(any()); + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService, org.mockito.Mockito.times(2)).record(auditCaptor.capture()); + assertThat(auditCaptor.getAllValues().get(0).metadata()) + .doesNotContainKey("estimated_bytes") + .containsEntry("outcome", "NO_ESTIMATE_REJECTED"); + } + + @Test + void executeRunsAMissingEstimateUnderRequireReviewAndRecordsTheCap() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + when(queryExecutor.execute(any())).thenReturn(new SelectExecutionResult( + List.of(new ResultColumn("id", 4, "int4")), List.of(List.of(1)), 1L, false, + Duration.ofMillis(5))); + + var outcome = service.execute(new ExecuteQueryCommand(queryId, submitterId, + organizationId, false)); + + assertThat(outcome.status()).isEqualTo(QueryStatus.EXECUTED); + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(auditCaptor.capture()); + assertThat(auditCaptor.getValue().metadata()) + .containsEntry("bytes_scanned_cap", 1_000L) + .containsEntry("bytes_scanned_cap_source", "GRANT") + .doesNotContainKey("bytes_scanned_estimate"); + } + + @Test + void executeWithinTheCapRecordsTheEstimateOnTheExecutedRow() { + when(queryRequestLookupService.findById(queryId)) + .thenReturn(Optional.of(snapshot(QueryStatus.APPROVED, QueryType.SELECT))); + givenBytesCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + givenBytesEstimate(10L); + when(queryExecutor.execute(any())).thenReturn(new SelectExecutionResult( + List.of(new ResultColumn("id", 4, "int4")), List.of(List.of(1)), 1L, false, + Duration.ofMillis(5))); + + service.execute(new ExecuteQueryCommand(queryId, submitterId, organizationId, false)); + + var auditCaptor = ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(auditCaptor.capture()); + assertThat(auditCaptor.getValue().metadata()).containsEntry("bytes_scanned_estimate", 10L); + } + @Test void executeRecordsFailureWhenExecutorThrows() { when(queryRequestLookupService.findById(queryId)) diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java index 243da3060..5945435eb 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryDecisionEvaluatorTest.java @@ -5,6 +5,8 @@ import com.bablsoft.accessflow.access.api.AccessGrantView; import com.bablsoft.accessflow.ai.api.BehaviorAnomalyLookupService; import com.bablsoft.accessflow.core.api.ApproverRule; +import com.bablsoft.accessflow.core.api.BytesScannedCapOutcome; +import com.bablsoft.accessflow.core.api.BytesScannedCapSource; import com.bablsoft.accessflow.core.api.QueryEstimateLookupService; import com.bablsoft.accessflow.core.api.QueryRequestLookupService; import com.bablsoft.accessflow.core.api.QueryRequestSnapshot; @@ -112,6 +114,8 @@ void aiFailureTraceSkipsEveryDecisionStage() { assertThat(trace.steps()).extracting("step", "outcome").containsExactly( org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.SQL_REVIEW, StepOutcome.NO_MATCH), + org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.BYTES_SCANNED_CAP, + StepOutcome.NO_MATCH), org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.ROUTING_POLICIES, StepOutcome.SKIP), org.assertj.core.groups.Tuple.tuple(QueryDecisionStepKind.GRANT_FAST_PATH, @@ -430,8 +434,9 @@ void thePlanFallThroughTraceCoversAllThreeStages() { 5, List.of(), clock).trace(); assertThat(trace.steps()).extracting("step").containsExactly( - QueryDecisionStepKind.SQL_REVIEW, QueryDecisionStepKind.ROUTING_POLICIES, - QueryDecisionStepKind.GRANT_FAST_PATH, QueryDecisionStepKind.REVIEW_PLAN); + QueryDecisionStepKind.SQL_REVIEW, QueryDecisionStepKind.BYTES_SCANNED_CAP, + QueryDecisionStepKind.ROUTING_POLICIES, QueryDecisionStepKind.GRANT_FAST_PATH, + QueryDecisionStepKind.REVIEW_PLAN); assertThat(step(trace, QueryDecisionStepKind.ROUTING_POLICIES).outcome()) .isEqualTo(StepOutcome.NO_MATCH); assertThat(step(trace, QueryDecisionStepKind.REVIEW_PLAN).outcome()).isEqualTo(StepOutcome.DENY); @@ -630,6 +635,159 @@ void theAiFailedPathStillRecordsTheBlockOnTheTrace() { assertThat(sqlReview.details()).containsEntry("blocking_rule_ids", List.of("select_star")); } + // ── Bytes-scanned cap (#941) ────────────────────────────────────────────── + + private static BytesCapCheck cap(Long estimated, BytesScannedCapOutcome outcome) { + return new BytesCapCheck(1_000_000_000_000L, BytesScannedCapSource.DATASOURCE, estimated, + outcome); + } + + @Test + void anExceededCapRejectsBeforeRoutingIsConsulted() { + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), + cap(2_000_000_000_000L, BytesScannedCapOutcome.EXCEEDED), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.BYTES_CAP_REJECTED); + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.REJECTED); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + assertThat(decision.context()).isNull(); + var capStep = step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP); + assertThat(capStep.outcome()).isEqualTo(StepOutcome.DENY); + assertThat(capStep.reasonKey()).isEqualTo("workflow.decision.bytes_cap.exceeded"); + assertThat(capStep.reasonArgs()).containsExactly( + "2 TB (2000000000000 B)", "1 TB (1000000000000 B)"); + assertThat(capStep.details()).containsEntry("bytes_scanned_cap_source", "DATASOURCE"); + assertThat(step(decision.trace(), QueryDecisionStepKind.ROUTING_POLICIES).reasonKey()) + .isEqualTo("workflow.decision.routing.skipped_bytes_cap"); + verify(routingPolicyEngine, never()).evaluate(any(), any(), any()); + verify(reviewPlanLookupService, never()).findForDatasource(any()); + } + + @Test + void aMissingEstimateUnderRejectRejectsEvenWhenAiFailed() { + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.FAILED, null, -1, + List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REJECTED), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.BYTES_CAP_REJECTED); + assertThat(step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP).reasonKey()) + .isEqualTo("workflow.decision.bytes_cap.no_estimate_rejected"); + } + + @Test + void anEstimateWithinTheCapChangesNothing() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of(), cap(5L, BytesScannedCapOutcome.WITHIN), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.PLAN_APPROVED); + assertThat(decision.bytesCap()).isNotNull(); + assertThat(decision.bytesCapChangedOutcome()).isFalse(); + assertThat(step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP).outcome()) + .isEqualTo(StepOutcome.ALLOW); + } + + @Test + void aMissingEstimateUnderRequireReviewSuppressesRoutingAutoApprove() { + givenPlan(false, true); + givenPolicyMatch(RoutingAction.AUTO_APPROVE, null); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), + clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.ROUTING_AUTO_APPROVE_SUPPRESSED); + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.PENDING_REVIEW); + assertThat(decision.sqlReviewSuppression()).isNull(); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + var routing = step(decision.trace(), QueryDecisionStepKind.ROUTING_POLICIES); + assertThat(routing.reasonKey()).isEqualTo( + "workflow.decision.routing.matched_auto_approve_suppressed_bytes_cap"); + assertThat(routing.details()).containsEntry("bytes_cap_suppressed", true) + .containsEntry("sql_review_suppressed", false); + assertThat(step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP).outcome()) + .isEqualTo(StepOutcome.MATCH); + } + + @Test + void aMissingEstimateUnderRequireReviewSuppressesTheGrantAndThePlan() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenActiveGrant(grant(true, false, false, List.of(), List.of())); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), clock); + + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.PENDING_REVIEW); + assertThat(decision.sqlReviewSuppression()).isNull(); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + assertThat(step(decision.trace(), QueryDecisionStepKind.GRANT_FAST_PATH).reasonKey()) + .isEqualTo("workflow.decision.grant.suppressed_bytes_cap"); + var plan = step(decision.trace(), QueryDecisionStepKind.REVIEW_PLAN); + assertThat(plan.reasonKey()).isEqualTo("workflow.decision.plan.suppressed_bytes_cap"); + assertThat(plan.details()).containsEntry("bytes_cap_suppressed", true); + } + + @Test + void aMissingEstimateOnARequestAlreadyHeadedToReviewChangesNothing() { + givenPlan(false, true); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.UPDATE), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), + clock); + + assertThat(decision.nextStatus()).isEqualTo(QueryStatus.PENDING_REVIEW); + assertThat(decision.bytesCapChangedOutcome()).isFalse(); + } + + @Test + void aMissingEstimateNeverSoftensAnAutoReject() { + givenPlan(false, true); + givenPolicyMatch(RoutingAction.AUTO_REJECT, null); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.COMPLETED, + RiskLevel.LOW, 5, List.of(), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), + clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.ROUTING_AUTO_REJECT); + assertThat(decision.bytesCapChangedOutcome()).isFalse(); + } + + @Test + void anUnevaluatedCapIsReportedWithoutDecidingAnything() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of(), cap(null, null), clock); + + assertThat(decision.kind()).isEqualTo(QueryDecisionKind.PLAN_APPROVED); + var capStep = step(decision.trace(), QueryDecisionStepKind.BYTES_SCANNED_CAP); + assertThat(capStep.outcome()).isEqualTo(StepOutcome.SKIP); + assertThat(capStep.reasonKey()).isEqualTo("workflow.decision.bytes_cap.unevaluated"); + } + + @Test + void theSqlReviewNamesTheSuppressionWhenBothGuardsApply() { + givenPlan(false, false); + givenNoPolicyMatch(); + givenNoGrants(); + + var decision = evaluator.evaluate(query(QueryType.SELECT), AiOutcome.SKIPPED, null, -1, + List.of("select_star"), cap(null, BytesScannedCapOutcome.NO_ESTIMATE_REVIEW), clock); + + assertThat(decision.sqlReviewSuppression()).isNotNull(); + assertThat(decision.bytesCapChangedOutcome()).isTrue(); + assertThat(step(decision.trace(), QueryDecisionStepKind.REVIEW_PLAN).reasonKey()) + .isEqualTo("workflow.decision.plan.suppressed_sql_review"); + } + // ── Fixtures ────────────────────────────────────────────────────────────── private static com.bablsoft.accessflow.core.api.DecisionTraceStep step( diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java index 869667c53..5afc2b887 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/QueryReviewStateMachineTest.java @@ -72,6 +72,9 @@ class QueryReviewStateMachineTest { @Mock AuditLogService auditLogService; @Mock MessageSource messageSource; @Mock ApplicationEventPublisher eventPublisher; + @Mock com.bablsoft.accessflow.core.api.BytesScannedCapResolutionService bytesScannedCapResolutionService; + @Mock com.bablsoft.accessflow.proxy.api.QueryCostEstimateService queryCostEstimateService; + @Mock org.springframework.transaction.PlatformTransactionManager transactionManager; // A real ConditionContextFactory over the same mocks, not a mock of it: the context builder is // what turns these signals into routing input, and mocking it away would stop testing that. @@ -97,7 +100,8 @@ void buildStateMachine() { sqlParserService, routingPolicyEngine, accessGrantLookupService); stateMachine = new QueryReviewStateMachine(queryRequestLookupService, evaluator, queryRequestStateService, routingDecisionService, sqlReviewFindingService, - auditLogService, messageSource, eventPublisher); + auditLogService, messageSource, eventPublisher, bytesScannedCapResolutionService, + queryCostEstimateService, queryEstimateLookupService, transactionManager); } @BeforeEach @@ -691,6 +695,176 @@ void anAuditWriteFailureDoesNotUndoTheTransition() { QueryStatus.PENDING_REVIEW); } + // ── Bytes-scanned cap (#941) ────────────────────────────────────────────── + + private void givenCap(long limit, + com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction missing) { + when(bytesScannedCapResolutionService.resolve(datasourceId, submitterId)) + .thenReturn(Optional.of(new com.bablsoft.accessflow.core.api.AppliedBytesCap(limit, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, missing))); + } + + private void givenEstimate(Long bytes, boolean failed) { + when(queryEstimateLookupService.findByQueryRequestId(queryId)).thenReturn(Optional.of( + new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), + queryId, "bigquery", QueryType.SELECT, !failed, null, null, null, null, + bytes, null, null, null, failed, null, 5, Instant.now()))); + } + + @Test + void aSkippedAnalysisWaitsForTheEstimateBeforeRouting() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, true, RiskLevel.LOW); + + stateMachine.onAiSkipped(new AiAnalysisSkippedEvent(queryId, "ai_analysis_enabled=false")); + + verify(queryCostEstimateService).estimateSubmittedQuery(queryId); + verify(queryRequestStateService, never()).recordBytesScannedCap(any(), org.mockito.ArgumentMatchers.anyLong(), + any(), any()); + } + + @Test + void aLostEstimateRaceIsReadBackRatherThanFailingTheDecision() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + // The listener won the insert: our own write fails, its row is what the cap compares. + when(queryCostEstimateService.estimateSubmittedQuery(queryId)) + .thenThrow(new org.springframework.dao.DataIntegrityViolationException("duplicate")); + givenEstimate(5_000L, false); + when(messageSource.getMessage(eq("workflow.bytes_cap.rejected_exceeded"), any(), any())) + .thenReturn("over the cap"); + + stateMachine.onAiSkipped(new AiAnalysisSkippedEvent(queryId, "ai_analysis_enabled=false")); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + verify(transactionManager).rollback(any()); + } + + @Test + void theAiCompletedPathOnlyEstimatesWhenACapApplies() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, true, RiskLevel.LOW); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryCostEstimateService, never()).estimateSubmittedQuery(any()); + } + + @Test + void anEstimateFailureOnTheSkippedPathIsAMissingEstimateNotAnError() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, true, RiskLevel.LOW); + when(queryCostEstimateService.estimateSubmittedQuery(queryId)) + .thenThrow(new IllegalStateException("warehouse down")); + + stateMachine.onAiSkipped(new AiAnalysisSkippedEvent(queryId, "ai_analysis_enabled=false")); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.PENDING_REVIEW); + } + + @Test + void anEstimateOverTheCapIsRejectedStampedAndAudited() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenEstimate(5_000L, false); + when(messageSource.getMessage(eq("workflow.bytes_cap.rejected_exceeded"), any(), any())) + .thenReturn("over the cap"); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED); + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + verify(eventPublisher).publishEvent(new QueryAutoRejectedEvent(queryId, null, "over the cap")); + var audit = org.mockito.ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(audit.capture()); + assertThat(audit.getValue().action()).isEqualTo(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED); + assertThat(audit.getValue().metadata()) + .containsEntry("stage", "decision") + .containsEntry("limit", 1_000L) + .containsEntry("estimated_bytes", 5_000L) + .containsEntry("outcome", "EXCEEDED") + .containsEntry("source", "DATASOURCE"); + verify(routingPolicyEngine, never()).evaluate(any(), any(), any()); + } + + @Test + void aFailedEstimateUnderRejectIsRejectedWithTheNoEstimateReason() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + givenEstimate(null, true); + when(messageSource.getMessage(eq("workflow.bytes_cap.rejected_no_estimate"), any(), any())) + .thenReturn("no estimate"); + + stateMachine.onAiFailed(new AiAnalysisFailedEvent(queryId, "provider error")); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + verify(eventPublisher).publishEvent(new QueryAutoRejectedEvent(queryId, null, "no estimate")); + var audit = org.mockito.ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(audit.capture()); + assertThat(audit.getValue().metadata()).doesNotContainKey("estimated_bytes") + .containsEntry("outcome", "NO_ESTIMATE_REJECTED"); + } + + @Test + void aMissingEstimateUnderRequireReviewHoldsAnAutoApprovalAndAuditsIt() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, false, RiskLevel.LOW); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.NO_ESTIMATE_REVIEW); + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.PENDING_REVIEW); + var audit = org.mockito.ArgumentCaptor.forClass(AuditEntry.class); + verify(auditLogService).record(audit.capture()); + assertThat(audit.getValue().action()).isEqualTo(AuditAction.QUERY_BYTES_SCANNED_CAP_ENFORCED); + } + + @Test + void anEstimateWithinTheCapIsStampedButNotAudited() { + givenPendingAiQuery(QueryType.SELECT); + givenPlan(false, false, RiskLevel.LOW); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REJECT); + givenEstimate(10L, false); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).recordBytesScannedCap(queryId, 1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.DATASOURCE, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.WITHIN); + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.APPROVED); + verify(auditLogService, never()).record(any()); + } + + @Test + void aFailingCapAuditNeverUndoesTheRejection() { + givenPendingAiQuery(QueryType.SELECT); + givenCap(1_000L, com.bablsoft.accessflow.core.api.BytesCapMissingEstimateAction.REQUIRE_REVIEW); + givenEstimate(5_000L, false); + org.mockito.Mockito.doThrow(new IllegalStateException("audit down")) + .when(auditLogService).record(any()); + + stateMachine.onAiCompleted(new AiAnalysisCompletedEvent(queryId, aiAnalysisId, + RiskLevel.LOW)); + + verify(queryRequestStateService).transitionTo(queryId, QueryStatus.PENDING_AI, + QueryStatus.REJECTED); + } + private void givenActiveGrant(AccessGrantView grant) { when(accessGrantLookupService.findActivePreApprovedGrants(organizationId, submitterId, datasourceId)).thenReturn(List.of(grant)); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java index b9fc2cfe8..a4f61cb8b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/ConditionContextFactoryTest.java @@ -69,7 +69,7 @@ private com.bablsoft.accessflow.core.api.QueryEstimateSnapshot estimate( Long estimatedRows, Long affectedRowCount, String scanType, boolean failed) { return new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot(UUID.randomUUID(), queryId, "postgresql", QueryType.SELECT, true, estimatedRows, affectedRowCount, - scanType, null, null, null, null, failed, null, 5, SUBMITTED_AT); + scanType, null, null, null, null, null, failed, null, 5, SUBMITTED_AT); } private QueryCorpusRow row(RiskLevel level, Integer score) { @@ -146,6 +146,23 @@ void historicalRowReplaysThePersistedCostEstimate() { assertThat(context.scanType()).isEqualTo("Seq Scan"); } + @Test + void theBytesEstimateIsReplayedAndAbsentOnAFailedEstimate() { + var withBytes = new com.bablsoft.accessflow.core.api.QueryEstimateSnapshot( + UUID.randomUUID(), queryId, "bigquery", QueryType.SELECT, true, 10L, null, null, + null, 7_000_000L, null, null, null, false, null, 5, SUBMITTED_AT); + when(queryEstimateLookupService.findByQueryRequestId(queryId)) + .thenReturn(Optional.of(withBytes)); + + assertThat(factory.forHistoricalRow(row(RiskLevel.LOW, 10), ZoneId.of("UTC")) + .estimatedBytesScanned()).isEqualTo(7_000_000L); + + when(queryEstimateLookupService.findByQueryRequestId(queryId)) + .thenReturn(Optional.of(estimate(5_000L, null, "Seq Scan", true))); + assertThat(factory.forHistoricalRow(row(RiskLevel.LOW, 10), ZoneId.of("UTC")) + .estimatedBytesScanned()).isNull(); + } + @Test void historicalRowPrefersTheExactAffectedRowCountOverThePlanEstimate() { when(queryEstimateLookupService.findByQueryRequestId(queryId)) diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java index 12515eafa..6ffdd7627 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionCodecTest.java @@ -52,6 +52,7 @@ void roundTripsEveryLeafAndCombinator() { new ConditionNode.TimeSinceLastApproval(ComparisonOperator.GT, 1440), new ConditionNode.CiCdOrigin(true), new ConditionNode.EstimatedRows(ComparisonOperator.GT, 100_000L), + new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, 1_000_000_000L), new ConditionNode.ScanTypeMatches(List.of("Seq*", "COLLSCAN")))); var json = codec.encode(tree); @@ -70,6 +71,17 @@ void estimatedRowsAndScanTypeUseSnakeCaseDiscriminators() { .contains("\"type\":\"scan_type\""); } + @Test + void estimatedBytesScannedUsesItsSnakeCaseDiscriminator() { + var json = codec.encode( + new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GTE, 1_000L)); + + assertThat(json).contains("\"type\":\"estimated_bytes_scanned\""); + assertThat(codec.decode( + "{\"type\":\"estimated_bytes_scanned\",\"operator\":\"GT\",\"value\":5}")) + .isEqualTo(new ConditionNode.EstimatedBytesScanned(ComparisonOperator.GT, 5L)); + } + @Test void queryShapeUsesItsDiscriminatorAndUpperCaseShapeNames() { var json = codec.encode(new ConditionNode.QueryShapeIn(Set.of(QueryShape.GROUP_BY))); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java index 8abd7b199..709a845cc 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/routing/RoutingConditionEvaluatorTest.java @@ -343,6 +343,28 @@ void estimatedRowsFailsClosedWhenNoEstimate() { new ConditionNode.EstimatedRows(ComparisonOperator.GTE, 0), ctx)).isFalse(); } + @Test + void estimatedBytesScanned() { + var ctx = bytesContext(2_000_000_000_000L); + assertThat(evaluator.matches(new ConditionNode.EstimatedBytesScanned( + ComparisonOperator.GT, 1_000_000_000_000L), ctx)).isTrue(); + assertThat(evaluator.matches(new ConditionNode.EstimatedBytesScanned( + ComparisonOperator.LTE, 1_000_000_000_000L), ctx)).isFalse(); + } + + @Test + void estimatedBytesScannedFailsClosedWhenNoEstimate() { + assertThat(evaluator.matches(new ConditionNode.EstimatedBytesScanned( + ComparisonOperator.GTE, 0), bytesContext(null))).isFalse(); + } + + private ConditionContext bytesContext(Long bytes) { + return new ConditionContext(QueryType.SELECT, Set.of("ds.events"), RiskLevel.LOW, 10, + "ANALYST", Set.of(groupId), LocalDateTime.of(2026, 6, 3, 14, 30), + false, false, false, null, null, false, null, false, 10L, null, Set.of(), false, + bytes); + } + @Test void scanType() { var ctx = estimateContext(500L, "Seq Scan"); diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java index 17188ff1f..f7e7d56be 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/AdminAccessSimulationControllerIntegrationTest.java @@ -119,7 +119,7 @@ void anAdminGetsTheFullOrderedTrace() { .exchange(); assertThat(result).hasStatus(200); - assertThat(result).bodyJson().extractingPath("$.steps.length()").asNumber().isEqualTo(12); + assertThat(result).bodyJson().extractingPath("$.steps.length()").asNumber().isEqualTo(13); assertThat(result).bodyJson().extractingPath("$.steps[0].step").asString() .isEqualTo("DATASOURCE_GATES"); // #864: the SQL review verdict sits between the permission gate and routing. @@ -127,7 +127,10 @@ void anAdminGetsTheFullOrderedTrace() { .isEqualTo("SQL_REVIEW"); assertThat(result).bodyJson().extractingPath("$.steps[4].outcome").asString() .isEqualTo("NO_MATCH"); - assertThat(result).bodyJson().extractingPath("$.steps[11].step").asString() + // #941: the bytes-scanned cap follows it; no cap configured here. + assertThat(result).bodyJson().extractingPath("$.steps[5].step").asString() + .isEqualTo("BYTES_SCANNED_CAP"); + assertThat(result).bodyJson().extractingPath("$.steps[12].step").asString() .isEqualTo("BREAK_GLASS"); assertThat(result).bodyJson().extractingPath("$.resulting_status").asString() .isEqualTo("PENDING_REVIEW"); @@ -292,11 +295,11 @@ void aDetailWithNoValueIsOmittedRatherThanSentAsNull() { .content(body(analyst.getId(), datasource.getId(), "SELECT id FROM orders")) .exchange(); - assertThat(result).bodyJson().extractingPath("$.steps[7].step").asString() + assertThat(result).bodyJson().extractingPath("$.steps[8].step").asString() .isEqualTo("REVIEW_PLAN"); - assertThat(result).bodyJson().doesNotHavePath("$.steps[7].details.review_plan_id"); - assertThat(result).bodyJson().doesNotHavePath("$.steps[7].details.min_approvals_required"); - assertThat(result).bodyJson().extractingPath("$.steps[7].details.requires_human_approval") + assertThat(result).bodyJson().doesNotHavePath("$.steps[8].details.review_plan_id"); + assertThat(result).bodyJson().doesNotHavePath("$.steps[8].details.min_approvals_required"); + assertThat(result).bodyJson().extractingPath("$.steps[8].details.requires_human_approval") .asBoolean().isFalse(); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java index 9c4d40922..518664c0b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/workflow/internal/web/QueryDetailResponseTest.java @@ -459,4 +459,34 @@ private QueryDetailView viewWithPrediction( null, Instant.now(), Instant.now()); } + + @Test + void fromCarriesTheBytesScannedCapAndTheBytesEstimate() { + var estimate = new QueryDetailView.CostEstimateDetail(UUID.randomUUID(), "bigquery", + QueryType.SELECT, true, null, null, null, null, null, null, null, false, null, 4, + 7_000L); + var base = new QueryDetailView(UUID.randomUUID(), UUID.randomUUID(), "Wh", DbType.BIGQUERY, + UUID.randomUUID(), UUID.randomUUID(), "a@x.io", "A", "SELECT 1", QueryType.SELECT, + QueryStatus.REJECTED, null, null, estimate, null, null, null, null, null, null, + null, null, null, null, List.of(), null, null, null, null, null, null, + Instant.now(), Instant.now(), null, null, null, null, + new QueryDetailView.BytesScannedCapDetail(1_000L, + com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED)); + + var response = QueryDetailResponse.from(base); + + assertThat(response.costEstimate().estimatedBytesScanned()).isEqualTo(7_000L); + assertThat(response.bytesScannedCap()).isEqualTo(new QueryDetailResponse.BytesScannedCapDetail( + 1_000L, com.bablsoft.accessflow.core.api.BytesScannedCapSource.GRANT, + com.bablsoft.accessflow.core.api.BytesScannedCapOutcome.EXCEEDED)); + } + + @Test + void theCompatibleCostEstimateConstructorLeavesTheBytesEstimateAbsent() { + var estimate = new QueryDetailView.CostEstimateDetail(UUID.randomUUID(), "postgresql", + QueryType.SELECT, true, 1L, null, null, null, null, null, null, false, null, 4); + + assertThat(estimate.estimatedBytesScanned()).isNull(); + } } From e3781cef5c65c906f882778bf91bc54a9f389ca4 Mon Sep 17 00:00:00 2001 From: Tigran Babloyan Date: Fri, 25 Sep 2026 13:20:25 +0400 Subject: [PATCH 2/4] feat(AF-941): bytes-scanned cap fields, routing operand and detail banner --- e2e/tests/admin-routing-policies.spec.ts | 84 ++++++++++++ frontend/src/components/common/BytesInput.tsx | 63 +++++++++ .../common/__tests__/BytesInput.test.tsx | 40 ++++++ .../policies/decisionTraceDetails.test.ts | 19 +++ .../policies/decisionTraceDetails.ts | 23 ++++ .../review/CostEstimatePanel.test.tsx | 16 +++ .../components/review/CostEstimatePanel.tsx | 13 +- frontend/src/locales/de.json | 53 ++++++-- frontend/src/locales/en.json | 53 ++++++-- frontend/src/locales/es.json | 53 ++++++-- frontend/src/locales/fr.json | 53 ++++++-- frontend/src/locales/hy.json | 53 ++++++-- frontend/src/locales/ru.json | 53 ++++++-- frontend/src/locales/zh-CN.json | 53 ++++++-- .../src/pages/admin/RoutingPoliciesPage.tsx | 16 +++ .../src/pages/admin/routingPolicyForm.test.ts | 33 +++++ frontend/src/pages/admin/routingPolicyForm.ts | 23 ++++ .../DatasourceCreateWizardPage.tsx | 55 +++++++- .../datasources/DatasourceSettingsPage.tsx | 101 +++++++++++++- .../DatasourceCreateWizardPage.test.tsx | 63 +++++++++ .../__tests__/DatasourceSettingsPage.test.tsx | 126 ++++++++++++++++++ .../pages/queries/QueryDetailPage.test.tsx | 51 +++++++ .../src/pages/queries/QueryDetailPage.tsx | 30 ++++- frontend/src/types/api.ts | 37 +++++ frontend/src/utils/__tests__/bytesCap.test.ts | 44 ++++++ .../__tests__/decisionTraceEnums.test.ts | 4 +- frontend/src/utils/apiErrors.ts | 1 + frontend/src/utils/bytesCap.ts | 57 ++++++++ frontend/src/utils/enumLabels.ts | 34 +++++ 29 files changed, 1240 insertions(+), 64 deletions(-) create mode 100644 frontend/src/components/common/BytesInput.tsx create mode 100644 frontend/src/components/common/__tests__/BytesInput.test.tsx create mode 100644 frontend/src/utils/__tests__/bytesCap.test.ts create mode 100644 frontend/src/utils/bytesCap.ts diff --git a/e2e/tests/admin-routing-policies.spec.ts b/e2e/tests/admin-routing-policies.spec.ts index ce7847f2b..ae2adf352 100644 --- a/e2e/tests/admin-routing-policies.spec.ts +++ b/e2e/tests/admin-routing-policies.spec.ts @@ -19,6 +19,8 @@ const BUILDER_POLICY_NAME = `Builder policy ${UNIQUE_SUFFIX}`; const AUTO_REJECT_POLICY_NAME = `Auto-reject deletes ${UNIQUE_SUFFIX}`; const CICD_REJECT_POLICY_NAME = `Block CI/CD ${UNIQUE_SUFFIX}`; const JOIN_REJECT_POLICY_NAME = `Block joins ${UNIQUE_SUFFIX}`; +const BYTES_BUILDER_POLICY_NAME = `Escalate big scans ${UNIQUE_SUFFIX}`; +const BYTES_REJECT_POLICY_NAME = `Block big scans ${UNIQUE_SUFFIX}`; const ROUTED_DS_NAME = `Routed DS ${UNIQUE_SUFFIX}`; const DEFAULT_API_BASE = 'http://localhost:8080'; @@ -205,4 +207,86 @@ test.describe.serial('/admin/routing-policies — routing engine', () => { ) .not.toMatch(/^(PENDING_AI|REJECTED)$/); }); + + // #941 — the builder offers the estimated_bytes_scanned operand and sends raw bytes. + test('builds an estimated-bytes-scanned condition entered in TB', async ({ page }) => { + await login(page, ADMIN_EMAIL, ADMIN_PASSWORD); + await page.goto('/admin/routing-policies'); + await waitForRoutingPoliciesListReady(page); + + await page.getByRole('button', { name: 'Add policy' }).first().click(); + const modal = page.getByRole('dialog').filter({ hasText: 'Add routing policy' }).first(); + await expect(modal).toBeVisible({ timeout: 10_000 }); + await modal.getByLabel('Name').fill(BYTES_BUILDER_POLICY_NAME); + await modal.getByLabel('Priority').fill(String(100_000 + Math.floor(Math.random() * 800_000))); + + // The seeded row is "Query type"; switch its operand. + await modal.locator('.ant-select').filter({ hasText: 'Query type' }).first().click(); + // The operand list is virtualised — off-screen options are not in the DOM — so walk it with + // the keyboard until the active option is the one we want. + const active = page.locator('.ant-select-item-option-active'); + for (let i = 0; i < 30; i += 1) { + if ((await active.textContent()) === 'Estimated bytes scanned') break; + await page.keyboard.press('ArrowDown'); + } + await expect(active).toHaveText('Estimated bytes scanned'); + await page.keyboard.press('Enter'); + const bytes = modal.getByLabel('Bytes scanned'); + await expect(bytes).toHaveValue('1'); + await bytes.fill('2'); + + const createResponse = page.waitForResponse( + (r) => + r.request().method() === 'POST' && + /\/api\/v1\/admin\/routing-policies$/.test(r.url()), + { timeout: 15_000 }, + ); + await modal.getByRole('button', { name: 'Create policy' }).click(); + const response = await createResponse; + expect(response.status()).toBe(201); + const created = (await response.json()) as CreatedRoutingPolicy & { + condition: { children?: { type: string; operator: string; value: number }[] }; + }; + createdPolicyIds.push(created.id); + expect(created.condition.children?.[0]).toEqual({ + type: 'estimated_bytes_scanned', + operator: 'GT', + value: 2_000_000_000_000, + }); + }); + + // #941 — PostgreSQL reports no bytes estimate, so the condition fails closed and never fires. + test('an estimated_bytes_scanned condition never fires without a bytes estimate', async ({ + request, + }) => { + const policy = await createRoutingPolicyViaApi(request, adminAccessToken, { + name: BYTES_REJECT_POLICY_NAME, + datasource_id: datasourceId as string, + priority: 100_000 + Math.floor(Math.random() * 800_000), + enabled: true, + action: 'AUTO_REJECT', + reason: 'scans are too large', + condition: { type: 'estimated_bytes_scanned', operator: 'GTE', value: 0 }, + }); + createdPolicyIds.push(policy.id); + + const query = await submitQueryViaApi( + request, + adminAccessToken, + datasourceId as string, + 'SELECT id FROM accounts WHERE id = 2', + 'e2e: bytes-scanned routing', + ); + await expect + .poll( + async () => { + const res = await request.get(`${apiBase()}/api/v1/queries/${query.id}`, { + headers: { Authorization: `Bearer ${adminAccessToken}` }, + }); + return ((await res.json()) as { status: string }).status; + }, + { timeout: 20_000 }, + ) + .not.toMatch(/^(PENDING_AI|REJECTED)$/); + }); }); diff --git a/frontend/src/components/common/BytesInput.tsx b/frontend/src/components/common/BytesInput.tsx new file mode 100644 index 000000000..508eb3470 --- /dev/null +++ b/frontend/src/components/common/BytesInput.tsx @@ -0,0 +1,63 @@ +import { useState } from 'react'; +import { InputNumber, Select, Space } from 'antd'; +import { useTranslation } from 'react-i18next'; +import { BYTE_INPUT_UNITS, byteUnitFactor, pickUnit, type ByteInputUnit } from '@/utils/bytesCap'; + +interface BytesInputProps { + /** Raw bytes; the form stores and submits bytes, the unit is display-only. */ + value?: number | null; + onChange?: (value: number | null) => void; + id?: string; + disabled?: boolean; + placeholder?: string; + 'aria-label'?: string; +} + +/** + * A byte count entered as an amount plus a unit (#941). Controlled through `value`/`onChange` so it + * drops into a `Form.Item` whose validation rules apply to the raw byte value. + */ +export function BytesInput({ + value, + onChange, + id, + disabled, + placeholder, + 'aria-label': ariaLabel, +}: BytesInputProps) { + const { t } = useTranslation(); + const [unit, setUnit] = useState(() => pickUnit(value)); + const factor = byteUnitFactor(unit); + const amount = value === null || value === undefined ? null : value / factor; + + const emit = (nextAmount: number | null, nextFactor: number) => { + onChange?.(nextAmount === null ? null : Math.round(nextAmount * nextFactor)); + }; + + return ( + + + id={id} + min={0} + step={1} + disabled={disabled} + placeholder={placeholder} + aria-label={ariaLabel} + value={amount} + onChange={(next) => emit(next, factor)} + style={{ width: '100%' }} + /> + + value={unit} + disabled={disabled} + aria-label={t('common.bytes_unit')} + style={{ width: 90 }} + options={BYTE_INPUT_UNITS.map((u) => ({ value: u.unit, label: u.unit }))} + onChange={(next) => { + setUnit(next); + emit(amount, byteUnitFactor(next)); + }} + /> + + ); +} diff --git a/frontend/src/components/common/__tests__/BytesInput.test.tsx b/frontend/src/components/common/__tests__/BytesInput.test.tsx new file mode 100644 index 000000000..b856e5da9 --- /dev/null +++ b/frontend/src/components/common/__tests__/BytesInput.test.tsx @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from 'vitest'; +import { fireEvent, render, screen } from '@testing-library/react'; +import '@/i18n'; +import { BytesInput } from '../BytesInput'; + +describe('BytesInput', () => { + it('shows a byte value in the largest unit it fills', () => { + render(); + expect(screen.getByLabelText('cap')).toHaveValue('2'); + expect(screen.getByText('TB')).toBeInTheDocument(); + }); + + it('emits raw bytes for the typed amount', () => { + const onChange = vi.fn(); + render(); + + fireEvent.change(screen.getByLabelText('cap'), { target: { value: '3' } }); + + expect(onChange).toHaveBeenLastCalledWith(3_000_000_000); + }); + + it('emits null when the amount is cleared', () => { + const onChange = vi.fn(); + render(); + + fireEvent.change(screen.getByLabelText('cap'), { target: { value: '' } }); + + expect(onChange).toHaveBeenLastCalledWith(null); + }); + + it('rescales the value when the unit changes', () => { + const onChange = vi.fn(); + render(); + + fireEvent.mouseDown(screen.getByRole('combobox')); + fireEvent.click(screen.getByTitle('TB')); + + expect(onChange).toHaveBeenLastCalledWith(2_000_000_000_000); + }); +}); diff --git a/frontend/src/components/policies/decisionTraceDetails.test.ts b/frontend/src/components/policies/decisionTraceDetails.test.ts index 84378f4fa..0e4bbcf94 100644 --- a/frontend/src/components/policies/decisionTraceDetails.test.ts +++ b/frontend/src/components/policies/decisionTraceDetails.test.ts @@ -186,4 +186,23 @@ describe('formatStepDetails — masking, omission and enum values (#1066 review) expect(byKey.scope).toBe('PIPELINE'); expect(rows.find((r) => r.key === 'approver_email')?.label).toBe('Approver'); }); + + it('formats the bytes-scanned cap step with byte sizes and labelled enums (#941)', () => { + const rows = formatStepDetails( + { + bytes_scanned_cap: 1_000_000_000_000, + bytes_scanned_cap_source: 'GRANT', + estimated_bytes_scanned: 2_500_000_000, + bytes_scanned_cap_outcome: 'EXCEEDED', + }, + t, + ); + const byKey = Object.fromEntries(rows.map((r) => [r.key, r.value])); + expect(byKey.bytes_scanned_cap).toBe('1 TB'); + expect(byKey.estimated_bytes_scanned).toBe('2.5 GB'); + expect(byKey.bytes_scanned_cap_source).toBe('Grant override'); + expect(byKey.bytes_scanned_cap_outcome).toBe( + 'Rejected: the estimate exceeds the bytes-scanned cap', + ); + }); }); diff --git a/frontend/src/components/policies/decisionTraceDetails.ts b/frontend/src/components/policies/decisionTraceDetails.ts index 2dc1294cd..55cb70050 100644 --- a/frontend/src/components/policies/decisionTraceDetails.ts +++ b/frontend/src/components/policies/decisionTraceDetails.ts @@ -1,5 +1,7 @@ import type { TFunction } from 'i18next'; import type { + BytesScannedCapOutcome, + BytesScannedCapSource, MaskingStrategy, QueryShape, QueryStatus, @@ -9,12 +11,17 @@ import type { RoutingPolicyTraceEntry, } from '@/types/api'; import { fmtDate } from '@/utils/dateFormat'; +import { formatBytes } from '@/utils/queryPlan'; import { + BYTES_SCANNED_CAP_OUTCOMES, + BYTES_SCANNED_CAP_SOURCES, MASKING_STRATEGIES, QUERY_SHAPES, QUERY_TYPES, RISK_LEVELS, ROUTING_ACTIONS, + bytesScannedCapOutcomeLabel, + bytesScannedCapSourceLabel, maskingStrategyLabel, queryShapeLabel, queryStatusLabel, @@ -47,6 +54,10 @@ export const KNOWN_DETAIL_KEYS = [ 'behavior', 'blocking_count', 'blocking_rule_ids', + 'bytes_cap_suppressed', + 'bytes_scanned_cap', + 'bytes_scanned_cap_outcome', + 'bytes_scanned_cap_source', 'can_break_glass', 'can_read', 'can_trigger', @@ -65,6 +76,7 @@ export const KNOWN_DETAIL_KEYS = [ 'environment_allows_break_glass', 'environment_name', 'environment_required_approvals', + 'estimated_bytes_scanned', 'estimated_rows', 'evaluated_at', 'expires_at', @@ -152,6 +164,17 @@ function enumValue(key: string, value: unknown, t: TFunction): string | null { } if (key === 'status' && includes(QUERY_STATUSES, value)) return queryStatusLabel(t, value); if (key === 'risk_level' && includes(RISK_LEVELS, value)) return riskLevelLabel(t, value); + if (key === 'bytes_scanned_cap_source' + && includes(BYTES_SCANNED_CAP_SOURCES, value)) { + return bytesScannedCapSourceLabel(t, value); + } + if (key === 'bytes_scanned_cap_outcome' + && includes(BYTES_SCANNED_CAP_OUTCOMES, value)) { + return bytesScannedCapOutcomeLabel(t, value); + } + if ((key === 'bytes_scanned_cap' || key === 'estimated_bytes_scanned') && typeof value === 'number') { + return formatBytes(value); + } return null; } diff --git a/frontend/src/components/review/CostEstimatePanel.test.tsx b/frontend/src/components/review/CostEstimatePanel.test.tsx index 6dca0eae4..5d2c51182 100644 --- a/frontend/src/components/review/CostEstimatePanel.test.tsx +++ b/frontend/src/components/review/CostEstimatePanel.test.tsx @@ -67,6 +67,22 @@ describe('CostEstimatePanel', () => { expect(screen.getByText('44,543.5')).toBeInTheDocument(); }); + it('renders the warehouse bytes-scanned estimate when present (#941)', () => { + render( + , + ); + expect(screen.getByText('Estimated bytes scanned')).toBeInTheDocument(); + expect(screen.getByText('1.5 TB')).toBeInTheDocument(); + }); + + it('omits the bytes line for engines that report none', () => { + render(); + expect(screen.queryByText('Estimated bytes scanned')).not.toBeInTheDocument(); + }); + it('renders the exact affected-row count for writes when present', () => { render( + {estimate.estimated_bytes_scanned !== null && + estimate.estimated_bytes_scanned !== undefined && ( +

+ {t('cost_estimate_panel.est_bytes_label')} + {formatBytes(estimate.estimated_bytes_scanned)} +
+ )} {estimate.scan_type && (
{t('cost_estimate_panel.scan_type_label')} diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.json index 4282e3447..e7278c1ab 100644 --- a/frontend/src/locales/de.json +++ b/frontend/src/locales/de.json @@ -36,7 +36,8 @@ "view_docs_aria": "Dokumentation anzeigen (wird in einem neuen Tab geöffnet)", "on_behalf_of": "im Auftrag von {{name}}", "on_behalf_of_tooltip": "Ein API-Key-Aufrufer hat für diese Person gehandelt (X-AccessFlow-On-Behalf-Of). Die genannte Person gilt für das Selbstgenehmigungsverbot als Einreicher.", - "principal_service_account_tooltip": "Eine nicht-menschliche Identität: sie authentifiziert sich nur per API-Key und kann sich nie interaktiv anmelden." + "principal_service_account_tooltip": "Eine nicht-menschliche Identität: sie authentifiziert sich nur per API-Key und kann sich nie interaktiv anmelden.", + "bytes_unit": "Einheit" }, "nav": { "editor": "SQL-Editor", @@ -654,7 +655,8 @@ "affected_rows_label": "Betroffene Zeilen (exakt)", "est_rows_label": "Geschätzte Zeilen", "scan_type_label": "Scan-Typ", - "cost_label": "Geschätzte Kosten" + "cost_label": "Geschätzte Kosten", + "est_bytes_label": "Geschätzte gescannte Bytes" }, "approval_prediction": { "value_label": "Historische Genehmigungswahrscheinlichkeit", @@ -864,7 +866,9 @@ "hint": "Effektives SQL ist die Anweisung, wie sie tatsächlich ausgeführt wurde, mit angewendeten Zeilensicherheits-Filtern und Soft-Delete-Umschreibungen. Gebundene Werte werden als ? angezeigt und nie gespeichert.", "diff_submitted_label": "Eingereichtes SQL", "diff_effective_label": "Effektives SQL" - } + }, + "bytes_cap_body": "Geschätzter Scan: {{estimate}} · Limit: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "keine Schätzung" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Umgebung", "environment_not_set": "Nicht gesetzt (Standardregeln der Organisation)", - "environment_help": "Legt fest, welcher SQL-Prüfregelsatz für Abfragen an dieser Datenquelle gilt." + "environment_help": "Legt fest, welcher SQL-Prüfregelsatz für Abfragen an dieser Datenquelle gilt.", + "label_max_bytes_scanned": "Max. gescannte Bytes pro Abfrage", + "max_bytes_scanned_help": "Abfragen ablehnen, deren Scan-Schätzung vor der Ausführung größer ist. Leer lassen für kein Limit.", + "max_bytes_scanned_placeholder": "Kein Limit", + "label_bytes_cap_missing_estimate": "Wenn keine Byte-Schätzung vorliegt", + "bytes_cap_missing_estimate_help": "Gilt nur, solange ein Limit gesetzt ist. „Menschliche Prüfung verlangen“ hält automatische Genehmigungen für eine Person zurück; „Ablehnen“ weist die Abfrage ab.", + "perm_col_bytes_cap": "Byte-Limit", + "perm_bytes_cap_none": "Standard", + "grant_bytes_cap_label": "Limit für gescannte Bytes", + "grant_bytes_cap_help": "Ein niedrigeres Limit für diesen Grant. Es gilt das strengste aus Datenquellen-Limit und allen Grants.", + "grant_bytes_cap_min": "Das Limit für gescannte Bytes muss mindestens 1 Byte betragen.", + "grant_bytes_cap_placeholder": "Standard der Datenquelle" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "z. B. *curl*, *GitHubActions*", "minutes_suffix": "Min.", "cicd_present_label": "Ist CI/CD-Herkunft", - "scan_type_placeholder": "z. B. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "z. B. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Gescannte Bytes" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Zeit seit letzter Genehmigung", "cicd_origin": "CI/CD-Herkunft", "estimated_rows": "Geschätzte Zeilen", - "scan_type": "Scan-Typ" + "scan_type": "Scan-Typ", + "estimated_bytes_scanned": "Geschätzte gescannte Bytes" }, "query_shape": { "JOIN": "Join", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Berechtigte Prüfer", "ROW_SECURITY": "Zeilensicherheit", "MASKING": "Maskierung", - "BREAK_GLASS": "Break-Glass" + "BREAK_GLASS": "Break-Glass", + "BYTES_SCANNED_CAP": "Limit für gescannte Bytes" }, "api_decision_step": { "CONNECTOR_GATES": "Konnektor-Prüfungen", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Feste Rate", "CRON": "Cron", "ONE_TIME": "Einmalig" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Menschliche Prüfung verlangen", + "REJECT": "Abfrage ablehnen" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Datenquellen-Limit", + "GRANT": "Grant-Override" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Innerhalb des Limits für gescannte Bytes", + "EXCEEDED": "Abgelehnt: die Schätzung überschreitet das Limit für gescannte Bytes", + "NO_ESTIMATE_REVIEW": "Zur Prüfung zurückgehalten: keine Byte-Schätzung unter dem Limit", + "NO_ESTIMATE_REJECTED": "Abgelehnt: keine Byte-Schätzung unter dem Limit" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "User-Agent des Anfragenden", "risk_level": "Risikostufe", "risk_score": "Risikowert", - "scan_type": "Scan-Typ" + "scan_type": "Scan-Typ", + "bytes_cap_suppressed": "Durch das Byte-Limit unterdrückt", + "bytes_scanned_cap": "Limit für gescannte Bytes", + "bytes_scanned_cap_outcome": "Ergebnis des Limits", + "bytes_scanned_cap_source": "Quelle des Limits", + "estimated_bytes_scanned": "Geschätzte gescannte Bytes" }, "use_id": "ID {{id}} verwenden", "user_id_placeholder": "Benutzer-ID einfügen", diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json index a344ddc38..569e80987 100644 --- a/frontend/src/locales/en.json +++ b/frontend/src/locales/en.json @@ -36,7 +36,8 @@ "view_docs_aria": "View docs (opens in a new tab)", "on_behalf_of": "on behalf of {{name}}", "on_behalf_of_tooltip": "An API-key caller acted for this person (X-AccessFlow-On-Behalf-Of). The named person is treated as a submitter for the self-approval ban.", - "principal_service_account_tooltip": "A non-human identity: it authenticates with an API key only and can never sign in interactively." + "principal_service_account_tooltip": "A non-human identity: it authenticates with an API key only and can never sign in interactively.", + "bytes_unit": "Unit" }, "nav": { "editor": "Query editor", @@ -654,7 +655,8 @@ "affected_rows_label": "Affected rows (exact)", "est_rows_label": "Estimated rows", "scan_type_label": "Scan type", - "cost_label": "Estimated cost" + "cost_label": "Estimated cost", + "est_bytes_label": "Estimated bytes scanned" }, "approval_prediction": { "value_label": "Historical approval likelihood", @@ -864,7 +866,9 @@ "hint": "Effective SQL is the statement as it actually ran, with row-security filters and soft-delete rewrites applied. Bound values are shown as ? and never stored.", "diff_submitted_label": "Submitted SQL", "diff_effective_label": "Effective SQL" - } + }, + "bytes_cap_body": "Estimated scan: {{estimate}} · Cap: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "no estimate" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Environment", "environment_not_set": "Not set (organization default rules)", - "environment_help": "Picks which SQL review ruleset applies to queries on this datasource." + "environment_help": "Picks which SQL review ruleset applies to queries on this datasource.", + "label_max_bytes_scanned": "Max bytes scanned per query", + "max_bytes_scanned_help": "Refuse queries whose pre-flight scan estimate is larger than this. Leave empty for no cap.", + "max_bytes_scanned_placeholder": "No cap", + "label_bytes_cap_missing_estimate": "When no bytes estimate is available", + "bytes_cap_missing_estimate_help": "Applies only while a cap is set. Require human review holds automatic approvals for a person; Reject refuses the query.", + "perm_col_bytes_cap": "Bytes cap", + "perm_bytes_cap_none": "default", + "grant_bytes_cap_label": "Bytes-scanned cap", + "grant_bytes_cap_help": "A lower cap for this grant. The strictest of the datasource cap and every grant applies.", + "grant_bytes_cap_min": "Bytes-scanned cap must be at least 1 byte.", + "grant_bytes_cap_placeholder": "Datasource default" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "e.g. *curl*, *GitHubActions*", "minutes_suffix": "min", "cicd_present_label": "Is CI/CD origin", - "scan_type_placeholder": "e.g. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "e.g. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Bytes scanned" }, "langfuse": { "title": "Langfuse", @@ -3300,7 +3316,8 @@ "time_since_last_approval": "Time since last approval", "cicd_origin": "CI/CD origin", "estimated_rows": "Estimated rows", - "scan_type": "Scan type" + "scan_type": "Scan type", + "estimated_bytes_scanned": "Estimated bytes scanned" }, "query_shape": { "JOIN": "Join", @@ -3740,7 +3757,8 @@ "ELIGIBLE_REVIEWERS": "Eligible reviewers", "ROW_SECURITY": "Row security", "MASKING": "Masking", - "BREAK_GLASS": "Break-glass" + "BREAK_GLASS": "Break-glass", + "BYTES_SCANNED_CAP": "Bytes-scanned cap" }, "api_decision_step": { "CONNECTOR_GATES": "Connector gates", @@ -3795,6 +3813,20 @@ "FIXED_RATE": "Fixed rate", "CRON": "Cron", "ONE_TIME": "One time" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Require human review", + "REJECT": "Reject the query" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Datasource cap", + "GRANT": "Grant override" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Within the bytes-scanned cap", + "EXCEEDED": "Rejected: the estimate exceeds the bytes-scanned cap", + "NO_ESTIMATE_REVIEW": "Held for review: no bytes estimate under the cap", + "NO_ESTIMATE_REJECTED": "Rejected: no bytes estimate under the cap" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Requester user agent", "risk_level": "Risk level", "risk_score": "Risk score", - "scan_type": "Scan type" + "scan_type": "Scan type", + "bytes_cap_suppressed": "Suppressed by the bytes cap", + "bytes_scanned_cap": "Bytes-scanned cap", + "bytes_scanned_cap_outcome": "Cap outcome", + "bytes_scanned_cap_source": "Cap source", + "estimated_bytes_scanned": "Estimated bytes scanned" }, "use_id": "Use ID {{id}}", "user_id_placeholder": "Paste a user ID", diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.json index 2f6b4d179..db432ca89 100644 --- a/frontend/src/locales/es.json +++ b/frontend/src/locales/es.json @@ -36,7 +36,8 @@ "view_docs_aria": "Ver documentación (se abre en una pestaña nueva)", "on_behalf_of": "en nombre de {{name}}", "on_behalf_of_tooltip": "Un llamador con clave de API actuó por esta persona (X-AccessFlow-On-Behalf-Of). La persona nombrada cuenta como solicitante para la prohibición de autoaprobación.", - "principal_service_account_tooltip": "Una identidad no humana: se autentica solo con una clave de API y nunca puede iniciar sesión de forma interactiva." + "principal_service_account_tooltip": "Una identidad no humana: se autentica solo con una clave de API y nunca puede iniciar sesión de forma interactiva.", + "bytes_unit": "Unidad" }, "nav": { "editor": "Editor SQL", @@ -654,7 +655,8 @@ "affected_rows_label": "Filas afectadas (exacto)", "est_rows_label": "Filas estimadas", "scan_type_label": "Tipo de escaneo", - "cost_label": "Coste estimado" + "cost_label": "Coste estimado", + "est_bytes_label": "Bytes escaneados estimados" }, "approval_prediction": { "value_label": "Probabilidad histórica de aprobación", @@ -864,7 +866,9 @@ "hint": "El SQL efectivo es la sentencia tal como se ejecutó realmente, con los filtros de seguridad de filas y las reescrituras de borrado lógico aplicados. Los valores enlazados se muestran como ? y nunca se almacenan.", "diff_submitted_label": "SQL enviado", "diff_effective_label": "SQL efectivo" - } + }, + "bytes_cap_body": "Escaneo estimado: {{estimate}} · Límite: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "sin estimación" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Entorno", "environment_not_set": "Sin definir (reglas predeterminadas de la organización)", - "environment_help": "Determina qué conjunto de reglas de revisión SQL se aplica a las consultas de esta fuente de datos." + "environment_help": "Determina qué conjunto de reglas de revisión SQL se aplica a las consultas de esta fuente de datos.", + "label_max_bytes_scanned": "Máximo de bytes escaneados por consulta", + "max_bytes_scanned_help": "Rechaza las consultas cuya estimación previa de escaneo sea mayor. Déjalo vacío para no poner límite.", + "max_bytes_scanned_placeholder": "Sin límite", + "label_bytes_cap_missing_estimate": "Cuando no hay estimación de bytes", + "bytes_cap_missing_estimate_help": "Solo se aplica mientras haya un límite. Exigir revisión humana retiene las aprobaciones automáticas para una persona; Rechazar deniega la consulta.", + "perm_col_bytes_cap": "Límite de bytes", + "perm_bytes_cap_none": "predeterminado", + "grant_bytes_cap_label": "Límite de bytes escaneados", + "grant_bytes_cap_help": "Un límite menor para este permiso. Se aplica el más estricto entre el límite de la fuente de datos y todos los permisos.", + "grant_bytes_cap_min": "El límite de bytes escaneados debe ser al menos 1 byte.", + "grant_bytes_cap_placeholder": "Predeterminado de la fuente de datos" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "p. ej. *curl*, *GitHubActions*", "minutes_suffix": "min", "cicd_present_label": "Es origen CI/CD", - "scan_type_placeholder": "p. ej. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "p. ej. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Bytes escaneados" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Tiempo desde la última aprobación", "cicd_origin": "Origen CI/CD", "estimated_rows": "Filas estimadas", - "scan_type": "Tipo de escaneo" + "scan_type": "Tipo de escaneo", + "estimated_bytes_scanned": "Bytes escaneados estimados" }, "query_shape": { "JOIN": "Join", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Revisores elegibles", "ROW_SECURITY": "Seguridad por filas", "MASKING": "Enmascaramiento", - "BREAK_GLASS": "Acceso de emergencia" + "BREAK_GLASS": "Acceso de emergencia", + "BYTES_SCANNED_CAP": "Límite de bytes escaneados" }, "api_decision_step": { "CONNECTOR_GATES": "Controles del conector", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Frecuencia fija", "CRON": "Cron", "ONE_TIME": "Única" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Exigir revisión humana", + "REJECT": "Rechazar la consulta" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Límite de la fuente de datos", + "GRANT": "Anulación del permiso" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Dentro del límite de bytes escaneados", + "EXCEEDED": "Rechazada: la estimación supera el límite de bytes escaneados", + "NO_ESTIMATE_REVIEW": "Retenida para revisión: sin estimación de bytes bajo el límite", + "NO_ESTIMATE_REJECTED": "Rechazada: sin estimación de bytes bajo el límite" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Agente de usuario del solicitante", "risk_level": "Nivel de riesgo", "risk_score": "Puntuación de riesgo", - "scan_type": "Tipo de escaneo" + "scan_type": "Tipo de escaneo", + "bytes_cap_suppressed": "Suprimido por el límite de bytes", + "bytes_scanned_cap": "Límite de bytes escaneados", + "bytes_scanned_cap_outcome": "Resultado del límite", + "bytes_scanned_cap_source": "Origen del límite", + "estimated_bytes_scanned": "Bytes escaneados estimados" }, "use_id": "Usar el ID {{id}}", "user_id_placeholder": "Pegue un ID de usuario", diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.json index d68360682..c36d0004c 100644 --- a/frontend/src/locales/fr.json +++ b/frontend/src/locales/fr.json @@ -36,7 +36,8 @@ "view_docs_aria": "Voir la documentation (s'ouvre dans un nouvel onglet)", "on_behalf_of": "pour le compte de {{name}}", "on_behalf_of_tooltip": "Un appelant par clé d'API a agi pour cette personne (X-AccessFlow-On-Behalf-Of). La personne nommée compte comme demandeur pour l'interdiction d'auto-approbation.", - "principal_service_account_tooltip": "Une identité non humaine : elle s'authentifie uniquement par clé d'API et ne peut jamais se connecter de façon interactive." + "principal_service_account_tooltip": "Une identité non humaine : elle s'authentifie uniquement par clé d'API et ne peut jamais se connecter de façon interactive.", + "bytes_unit": "Unité" }, "nav": { "editor": "Éditeur SQL", @@ -654,7 +655,8 @@ "affected_rows_label": "Lignes affectées (exact)", "est_rows_label": "Lignes estimées", "scan_type_label": "Type de scan", - "cost_label": "Coût estimé" + "cost_label": "Coût estimé", + "est_bytes_label": "Octets analysés estimés" }, "approval_prediction": { "value_label": "Probabilité d'approbation historique", @@ -864,7 +866,9 @@ "hint": "Le SQL effectif est l’instruction telle qu’elle a réellement été exécutée, avec les filtres de sécurité des lignes et les réécritures de suppression logique appliqués. Les valeurs liées sont affichées sous forme de ? et ne sont jamais stockées.", "diff_submitted_label": "SQL soumis", "diff_effective_label": "SQL effectif" - } + }, + "bytes_cap_body": "Analyse estimée : {{estimate}} · Limite : {{limit}} ({{source}})", + "bytes_cap_no_estimate": "aucune estimation" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Environnement", "environment_not_set": "Non défini (règles par défaut de l'organisation)", - "environment_help": "Détermine quel jeu de règles de revue SQL s’applique aux requêtes sur cette source de données." + "environment_help": "Détermine quel jeu de règles de revue SQL s’applique aux requêtes sur cette source de données.", + "label_max_bytes_scanned": "Octets analysés max. par requête", + "max_bytes_scanned_help": "Refuse les requêtes dont l’estimation préalable dépasse cette valeur. Laisser vide pour aucune limite.", + "max_bytes_scanned_placeholder": "Aucune limite", + "label_bytes_cap_missing_estimate": "Sans estimation en octets", + "bytes_cap_missing_estimate_help": "S’applique seulement si une limite est définie. « Exiger une revue humaine » soumet les approbations automatiques à une personne ; « Rejeter » refuse la requête.", + "perm_col_bytes_cap": "Limite d’octets", + "perm_bytes_cap_none": "par défaut", + "grant_bytes_cap_label": "Limite d’octets analysés", + "grant_bytes_cap_help": "Une limite plus basse pour cette autorisation. La plus stricte entre la limite de la source de données et toutes les autorisations s’applique.", + "grant_bytes_cap_min": "La limite d’octets analysés doit être d’au moins 1 octet.", + "grant_bytes_cap_placeholder": "Valeur par défaut de la source de données" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "p. ex. *curl*, *GitHubActions*", "minutes_suffix": "min", "cicd_present_label": "Provenance CI/CD", - "scan_type_placeholder": "p. ex. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "p. ex. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Octets analysés" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Temps depuis la dernière approbation", "cicd_origin": "Provenance CI/CD", "estimated_rows": "Lignes estimées", - "scan_type": "Type de scan" + "scan_type": "Type de scan", + "estimated_bytes_scanned": "Octets analysés estimés" }, "query_shape": { "JOIN": "Jointure", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Réviseurs éligibles", "ROW_SECURITY": "Sécurité par ligne", "MASKING": "Masquage", - "BREAK_GLASS": "Accès d'urgence" + "BREAK_GLASS": "Accès d'urgence", + "BYTES_SCANNED_CAP": "Limite d’octets analysés" }, "api_decision_step": { "CONNECTOR_GATES": "Contrôles du connecteur", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Fréquence fixe", "CRON": "Cron", "ONE_TIME": "Unique" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Exiger une revue humaine", + "REJECT": "Rejeter la requête" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Limite de la source de données", + "GRANT": "Dérogation de l’autorisation" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Dans la limite d’octets analysés", + "EXCEEDED": "Rejetée : l’estimation dépasse la limite d’octets analysés", + "NO_ESTIMATE_REVIEW": "Mise en revue : aucune estimation en octets sous la limite", + "NO_ESTIMATE_REJECTED": "Rejetée : aucune estimation en octets sous la limite" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Agent utilisateur du demandeur", "risk_level": "Niveau de risque", "risk_score": "Score de risque", - "scan_type": "Type de parcours" + "scan_type": "Type de parcours", + "bytes_cap_suppressed": "Neutralisé par la limite d’octets", + "bytes_scanned_cap": "Limite d’octets analysés", + "bytes_scanned_cap_outcome": "Résultat de la limite", + "bytes_scanned_cap_source": "Origine de la limite", + "estimated_bytes_scanned": "Octets analysés estimés" }, "use_id": "Utiliser l'ID {{id}}", "user_id_placeholder": "Collez un ID d'utilisateur", diff --git a/frontend/src/locales/hy.json b/frontend/src/locales/hy.json index 582d46bfc..c20dfd2ef 100644 --- a/frontend/src/locales/hy.json +++ b/frontend/src/locales/hy.json @@ -36,7 +36,8 @@ "view_docs_aria": "Դիտել փաստաթղթերը (բացվում է նոր ներդիրում)", "on_behalf_of": "{{name}}-ի անունից", "on_behalf_of_tooltip": "API բանալիով զանգողը գործել է այս անձի անունից (X-AccessFlow-On-Behalf-Of)։ Նշված անձը ինքնահաստատման արգելքի համար համարվում է ներկայացնող։", - "principal_service_account_tooltip": "Ոչ մարդկային ինքնություն. նույնականանում է միայն API բանալիով և երբեք չի կարող ինտերակտիվ մուտք գործել։" + "principal_service_account_tooltip": "Ոչ մարդկային ինքնություն. նույնականանում է միայն API բանալիով և երբեք չի կարող ինտերակտիվ մուտք գործել։", + "bytes_unit": "Միավոր" }, "nav": { "editor": "SQL խմբագիր", @@ -654,7 +655,8 @@ "affected_rows_label": "Ազդված տողեր (ճշգրիտ)", "est_rows_label": "Գնահատված տողեր", "scan_type_label": "Սկանավորման տեսակ", - "cost_label": "Գնահատված արժեք" + "cost_label": "Գնահատված արժեք", + "est_bytes_label": "Սկանավորվող բայթերի գնահատական" }, "approval_prediction": { "value_label": "Հաստատման պատմական հավանականություն", @@ -864,7 +866,9 @@ "hint": "Փաստացի SQL-ը հրահանգն է այնպես, ինչպես այն իրականում կատարվել է՝ տողերի անվտանգության զտիչներով և փափուկ ջնջման վերաշարադրումներով։ Կապված արժեքները ցուցադրվում են որպես ? և երբեք չեն պահպանվում։", "diff_submitted_label": "Ներկայացված SQL", "diff_effective_label": "Փաստացի SQL" - } + }, + "bytes_cap_body": "Գնահատված սկանավորում՝ {{estimate}} · Սահմանաչափ՝ {{limit}} ({{source}})", + "bytes_cap_no_estimate": "գնահատական չկա" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Միջավայր", "environment_not_set": "Սահմանված չէ (կազմակերպության լռելյայն կանոններ)", - "environment_help": "Որոշում է, թե որ SQL ստուգման կանոնակարգն է կիրառվում այս տվյալների աղբյուրի հարցումների վրա։" + "environment_help": "Որոշում է, թե որ SQL ստուգման կանոնակարգն է կիրառվում այս տվյալների աղբյուրի հարցումների վրա։", + "label_max_bytes_scanned": "Սկանավորված բայթերի առավելագույնը մեկ հարցման համար", + "max_bytes_scanned_help": "Մերժել հարցումները, որոնց նախնական սկանավորման գնահատականն ավելի մեծ է։ Թողեք դատարկ՝ առանց սահմանաչափի։", + "max_bytes_scanned_placeholder": "Առանց սահմանաչափի", + "label_bytes_cap_missing_estimate": "Երբ բայթերի գնահատական չկա", + "bytes_cap_missing_estimate_help": "Կիրառվում է միայն սահմանաչափ սահմանված լինելու դեպքում։ «Պահանջել մարդկային ստուգում»-ը ավտոմատ հաստատումները թողնում է մարդուն, «Մերժել»-ը մերժում է հարցումը։", + "perm_col_bytes_cap": "Բայթերի սահմանաչափ", + "perm_bytes_cap_none": "լռելյայն", + "grant_bytes_cap_label": "Սկանավորված բայթերի սահմանաչափ", + "grant_bytes_cap_help": "Ավելի ցածր սահմանաչափ այս թույլտվության համար։ Կիրառվում է տվյալների աղբյուրի և բոլոր թույլտվությունների սահմանաչափերից ամենախիստը։", + "grant_bytes_cap_min": "Սկանավորված բայթերի սահմանաչափը պետք է լինի առնվազն 1 բայթ։", + "grant_bytes_cap_placeholder": "Տվյալների աղբյուրի լռելյայնը" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "օր. *curl*, *GitHubActions*", "minutes_suffix": "ր", "cicd_present_label": "CI/CD ծագում", - "scan_type_placeholder": "օր.՝ Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "օր.՝ Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Սկանավորված բայթեր" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Ժամանակ վերջին հաստատումից", "cicd_origin": "CI/CD ծագում", "estimated_rows": "Գնահատված տողեր", - "scan_type": "Սկանավորման տեսակ" + "scan_type": "Սկանավորման տեսակ", + "estimated_bytes_scanned": "Սկանավորվող բայթերի գնահատական" }, "query_shape": { "JOIN": "Միացում (JOIN)", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Իրավասու վերանայողներ", "ROW_SECURITY": "Տողային անվտանգություն", "MASKING": "Քողարկում", - "BREAK_GLASS": "Արտակարգ մուտք" + "BREAK_GLASS": "Արտակարգ մուտք", + "BYTES_SCANNED_CAP": "Սկանավորված բայթերի սահմանաչափ" }, "api_decision_step": { "CONNECTOR_GATES": "Միակցիչի ստուգումներ", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Ֆիքսված հաճախականություն", "CRON": "Cron", "ONE_TIME": "Միանվագ" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Պահանջել մարդկային ստուգում", + "REJECT": "Մերժել հարցումը" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Տվյալների աղբյուրի սահմանաչափ", + "GRANT": "Թույլտվության վերասահմանում" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "Սկանավորված բայթերի սահմանաչափի մեջ է", + "EXCEEDED": "Մերժված է. գնահատականը գերազանցում է սկանավորված բայթերի սահմանաչափը", + "NO_ESTIMATE_REVIEW": "Պահված է ստուգման համար. սահմանաչափի ներքո բայթերի գնահատական չկա", + "NO_ESTIMATE_REJECTED": "Մերժված է. սահմանաչափի ներքո բայթերի գնահատական չկա" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "Հայցողի user agent", "risk_level": "Ռիսկի մակարդակ", "risk_score": "Ռիսկի միավոր", - "scan_type": "Սկանավորման տեսակ" + "scan_type": "Սկանավորման տեսակ", + "bytes_cap_suppressed": "Ճնշված է բայթերի սահմանաչափով", + "bytes_scanned_cap": "Սկանավորված բայթերի սահմանաչափ", + "bytes_scanned_cap_outcome": "Սահմանաչափի արդյունք", + "bytes_scanned_cap_source": "Սահմանաչափի աղբյուր", + "estimated_bytes_scanned": "Սկանավորվող բայթերի գնահատական" }, "use_id": "Օգտագործել ID {{id}}", "user_id_placeholder": "Տեղադրեք օգտատիրոջ ID", diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.json index fec35742c..5367102af 100644 --- a/frontend/src/locales/ru.json +++ b/frontend/src/locales/ru.json @@ -36,7 +36,8 @@ "view_docs_aria": "Открыть документацию (в новой вкладке)", "on_behalf_of": "от имени {{name}}", "on_behalf_of_tooltip": "Вызывающий с API-ключом действовал от имени этого человека (X-AccessFlow-On-Behalf-Of). Названный человек считается отправителем для запрета самоодобрения.", - "principal_service_account_tooltip": "Нечеловеческая идентичность: аутентифицируется только API-ключом и никогда не может войти интерактивно." + "principal_service_account_tooltip": "Нечеловеческая идентичность: аутентифицируется только API-ключом и никогда не может войти интерактивно.", + "bytes_unit": "Единица" }, "nav": { "editor": "Редактор SQL", @@ -654,7 +655,8 @@ "affected_rows_label": "Затронутые строки (точно)", "est_rows_label": "Оценка строк", "scan_type_label": "Тип сканирования", - "cost_label": "Оценочная стоимость" + "cost_label": "Оценочная стоимость", + "est_bytes_label": "Оценка сканируемых байтов" }, "approval_prediction": { "value_label": "Историческая вероятность одобрения", @@ -864,7 +866,9 @@ "hint": "Фактический SQL — это запрос в том виде, в котором он был выполнен, с применёнными фильтрами построчной безопасности и перезаписью мягкого удаления. Связанные значения показаны как ? и никогда не сохраняются.", "diff_submitted_label": "Отправленный SQL", "diff_effective_label": "Фактический SQL" - } + }, + "bytes_cap_body": "Оценка сканирования: {{estimate}} · Лимит: {{limit}} ({{source}})", + "bytes_cap_no_estimate": "нет оценки" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "Среда", "environment_not_set": "Не задано (правила организации по умолчанию)", - "environment_help": "Определяет, какой набор правил проверки SQL применяется к запросам к этому источнику данных." + "environment_help": "Определяет, какой набор правил проверки SQL применяется к запросам к этому источнику данных.", + "label_max_bytes_scanned": "Макс. сканируемых байтов на запрос", + "max_bytes_scanned_help": "Отклонять запросы, чья предварительная оценка сканирования больше. Оставьте пустым, чтобы не ограничивать.", + "max_bytes_scanned_placeholder": "Без лимита", + "label_bytes_cap_missing_estimate": "Если оценки в байтах нет", + "bytes_cap_missing_estimate_help": "Действует только при заданном лимите. «Требовать проверку человеком» оставляет автоматические одобрения человеку; «Отклонить» отклоняет запрос.", + "perm_col_bytes_cap": "Лимит байтов", + "perm_bytes_cap_none": "по умолчанию", + "grant_bytes_cap_label": "Лимит сканируемых байтов", + "grant_bytes_cap_help": "Более низкий лимит для этой выдачи доступа. Применяется самый строгий из лимита источника данных и всех выдач.", + "grant_bytes_cap_min": "Лимит сканируемых байтов должен быть не менее 1 байта.", + "grant_bytes_cap_placeholder": "По умолчанию источника данных" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "напр. *curl*, *GitHubActions*", "minutes_suffix": "мин", "cicd_present_label": "Источник CI/CD", - "scan_type_placeholder": "напр. Seq Scan, COLLSCAN, Index*" + "scan_type_placeholder": "напр. Seq Scan, COLLSCAN, Index*", + "bytes_value_label": "Сканируемые байты" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "Время с последнего одобрения", "cicd_origin": "Источник CI/CD", "estimated_rows": "Оценка строк", - "scan_type": "Тип сканирования" + "scan_type": "Тип сканирования", + "estimated_bytes_scanned": "Оценка сканируемых байтов" }, "query_shape": { "JOIN": "Соединение (JOIN)", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "Допустимые проверяющие", "ROW_SECURITY": "Построчная безопасность", "MASKING": "Маскирование", - "BREAK_GLASS": "Экстренный доступ" + "BREAK_GLASS": "Экстренный доступ", + "BYTES_SCANNED_CAP": "Лимит сканируемых байтов" }, "api_decision_step": { "CONNECTOR_GATES": "Проверки коннектора", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "Фиксированная частота", "CRON": "Cron", "ONE_TIME": "Однократно" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "Требовать проверку человеком", + "REJECT": "Отклонить запрос" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "Лимит источника данных", + "GRANT": "Переопределение выдачи доступа" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "В пределах лимита сканируемых байтов", + "EXCEEDED": "Отклонено: оценка превышает лимит сканируемых байтов", + "NO_ESTIMATE_REVIEW": "Отправлено на проверку: нет оценки в байтах при лимите", + "NO_ESTIMATE_REJECTED": "Отклонено: нет оценки в байтах при лимите" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "User agent автора", "risk_level": "Уровень риска", "risk_score": "Оценка риска", - "scan_type": "Тип сканирования" + "scan_type": "Тип сканирования", + "bytes_cap_suppressed": "Подавлено лимитом байтов", + "bytes_scanned_cap": "Лимит сканируемых байтов", + "bytes_scanned_cap_outcome": "Результат лимита", + "bytes_scanned_cap_source": "Источник лимита", + "estimated_bytes_scanned": "Оценка сканируемых байтов" }, "use_id": "Использовать ID {{id}}", "user_id_placeholder": "Вставьте ID пользователя", diff --git a/frontend/src/locales/zh-CN.json b/frontend/src/locales/zh-CN.json index 11dfabd35..ea8a8cb5f 100644 --- a/frontend/src/locales/zh-CN.json +++ b/frontend/src/locales/zh-CN.json @@ -36,7 +36,8 @@ "view_docs_aria": "查看文档（在新标签页中打开）", "on_behalf_of": "代表 {{name}}", "on_behalf_of_tooltip": "某个 API 密钥调用方代表此人执行了操作（X-AccessFlow-On-Behalf-Of）。在禁止自行审批的规则中，被指名的人视为提交者。", - "principal_service_account_tooltip": "非人类身份：仅通过 API 密钥认证，永远无法交互式登录。" + "principal_service_account_tooltip": "非人类身份：仅通过 API 密钥认证，永远无法交互式登录。", + "bytes_unit": "单位" }, "nav": { "editor": "SQL 编辑器", @@ -654,7 +655,8 @@ "affected_rows_label": "受影响行数（精确）", "est_rows_label": "预估行数", "scan_type_label": "扫描类型", - "cost_label": "预估成本" + "cost_label": "预估成本", + "est_bytes_label": "预计扫描字节数" }, "approval_prediction": { "value_label": "历史批准可能性", @@ -864,7 +866,9 @@ "hint": "实际执行的 SQL 是语句真正运行时的形式，已应用行级安全过滤和软删除改写。绑定值显示为 ?，且从不存储。", "diff_submitted_label": "提交的 SQL", "diff_effective_label": "实际执行的 SQL" - } + }, + "bytes_cap_body": "预计扫描：{{estimate}} · 上限：{{limit}}（{{source}}）", + "bytes_cap_no_estimate": "无估算" } }, "reviews": { @@ -1575,7 +1579,18 @@ }, "label_environment": "环境", "environment_not_set": "未设置（使用组织默认规则）", - "environment_help": "决定对此数据源的查询应用哪个 SQL 审查规则集。" + "environment_help": "决定对此数据源的查询应用哪个 SQL 审查规则集。", + "label_max_bytes_scanned": "每次查询最大扫描字节数", + "max_bytes_scanned_help": "拒绝预检扫描估算大于此值的查询。留空表示不设上限。", + "max_bytes_scanned_placeholder": "无上限", + "label_bytes_cap_missing_estimate": "没有字节估算时", + "bytes_cap_missing_estimate_help": "仅在设置了上限时生效。“要求人工审查”会把自动批准交给人工处理；“拒绝”会拒绝查询。", + "perm_col_bytes_cap": "字节上限", + "perm_bytes_cap_none": "默认", + "grant_bytes_cap_label": "扫描字节上限", + "grant_bytes_cap_help": "为此授权设置更低的上限。取数据源上限与所有授权中最严格的一个。", + "grant_bytes_cap_min": "扫描字节上限至少为 1 字节。", + "grant_bytes_cap_placeholder": "数据源默认值" } }, "admin": { @@ -2455,7 +2470,8 @@ "user_agent_placeholder": "例如 *curl*、*GitHubActions*", "minutes_suffix": "分钟", "cicd_present_label": "来自 CI/CD", - "scan_type_placeholder": "例如 Seq Scan、COLLSCAN、Index*" + "scan_type_placeholder": "例如 Seq Scan、COLLSCAN、Index*", + "bytes_value_label": "扫描字节数" }, "langfuse": { "title": "Langfuse", @@ -3270,7 +3286,8 @@ "time_since_last_approval": "距上次批准时长", "cicd_origin": "CI/CD 来源", "estimated_rows": "预估行数", - "scan_type": "扫描类型" + "scan_type": "扫描类型", + "estimated_bytes_scanned": "预计扫描字节数" }, "query_shape": { "JOIN": "连接（JOIN）", @@ -3715,7 +3732,8 @@ "ELIGIBLE_REVIEWERS": "合格审查人", "ROW_SECURITY": "行级安全", "MASKING": "脱敏", - "BREAK_GLASS": "紧急访问" + "BREAK_GLASS": "紧急访问", + "BYTES_SCANNED_CAP": "扫描字节上限" }, "api_decision_step": { "CONNECTOR_GATES": "连接器检查", @@ -3770,6 +3788,20 @@ "FIXED_RATE": "固定频率", "CRON": "Cron", "ONE_TIME": "一次性" + }, + "bytes_cap_missing_estimate": { + "REQUIRE_REVIEW": "要求人工审查", + "REJECT": "拒绝查询" + }, + "bytes_scanned_cap_source": { + "DATASOURCE": "数据源上限", + "GRANT": "授权覆盖" + }, + "bytes_scanned_cap_outcome": { + "WITHIN": "在扫描字节上限之内", + "EXCEEDED": "已拒绝：估算超过扫描字节上限", + "NO_ESTIMATE_REVIEW": "已转人工审查：上限下没有字节估算", + "NO_ESTIMATE_REJECTED": "已拒绝：上限下没有字节估算" } }, "access": { @@ -5775,7 +5807,12 @@ "requester_user_agent": "请求者 User-Agent", "risk_level": "风险等级", "risk_score": "风险分数", - "scan_type": "扫描类型" + "scan_type": "扫描类型", + "bytes_cap_suppressed": "被字节上限抑制", + "bytes_scanned_cap": "扫描字节上限", + "bytes_scanned_cap_outcome": "上限结果", + "bytes_scanned_cap_source": "上限来源", + "estimated_bytes_scanned": "预计扫描字节数" }, "use_id": "使用 ID {{id}}", "user_id_placeholder": "粘贴用户 ID", diff --git a/frontend/src/pages/admin/RoutingPoliciesPage.tsx b/frontend/src/pages/admin/RoutingPoliciesPage.tsx index 72607cd0f..6bdbc4b9c 100644 --- a/frontend/src/pages/admin/RoutingPoliciesPage.tsx +++ b/frontend/src/pages/admin/RoutingPoliciesPage.tsx @@ -33,6 +33,7 @@ import { } from '@/components/policies/policyImpact'; import { simulateRoutingPolicy } from '@/api/policySimulation'; import { routingSimulationSummary } from './routingSimulationSummary'; +import { BytesInput } from '@/components/common/BytesInput'; import { PageHeader } from '@/components/common/PageHeader'; import { EmptyState } from '@/components/common/EmptyState'; import { Pill } from '@/components/common/Pill'; @@ -942,6 +943,21 @@ function ConditionValueEditor({ name, operand, groups, roleOptions }: ConditionV
); + case 'estimated_bytes_scanned': + return ( +
+ + ({ + value: v, + label: bytesCapMissingEstimateLabel(t, v), + }))} + /> + + + )} @@ -499,9 +508,21 @@ function ConfigTab({ ds, onDelete, deletePending }: ConfigTabProps) { }); const onFinish = (values: SettingsFormValues) => { - const { read_replicas: replicaRows, environment, ...rest } = values; + const { + read_replicas: replicaRows, + environment, + max_bytes_scanned_per_query: bytesCap, + bytes_cap_missing_estimate: bytesCapMissingEstimate, + ...rest + } = values; // "Not set" must clear explicitly: a null environment means "unchanged" to the API (#861). const body: UpdateDatasourceInput = { ...rest, ...toEnvironmentUpdate(environment) }; + // The cap fields exist only on bytes-reporting engines; anywhere else they are never sent (#941). + if (bytesCapSupported) { + Object.assign(body, toBytesCapUpdate(bytesCap, ds.max_bytes_scanned_per_query), { + bytes_cap_missing_estimate: bytesCapMissingEstimate, + }); + } if (!body.password || body.password.trim().length === 0) { delete body.password; } @@ -803,6 +824,37 @@ function ConfigTab({ ds, onDelete, deletePending }: ConfigTabProps) { }))} /> + {bytesCapSupported && ( + <> + {/* Optional; mirrors the backend @Min(1) (#941). Empty = no cap. */} + + + + +