Add authenticated database readiness check for Astra and Cassandra

CONFIGURATION.md

@@ -54,6 +54,48 @@ Other Quarkus properties that are specifically relevant for the service:
 | `stargate.jsonapi.operations.database-config.ddl-delay-millis`          | `int`     | `2000`   | Delay between create table and create index to get the schema sync.                                                                                                                                |
 | `stargate.jsonapi.operations.vectorize-enabled`                         | `boolean` | `false`  | Flag to enable server side vectorization.                                                                                                                                              |
 
+### Database readiness
+
+`GET /v1/health/ready` is an authenticated database readiness endpoint used for both Astra and
+Cassandra deployments. It uses the request's tenant, `Token` header, and `User-Agent` to obtain a
+session through the normal session cache. The Data API does not store separate readiness
+credentials.
+
+The endpoint executes `SELECT * FROM datastax_sla.check LIMIT 1` at `LOCAL_QUORUM`, using the
+`table-read` driver profile for the remaining read settings. An `UP` response therefore confirms
+that the coordinator can complete a read from a replicated table at local quorum. It does not
+validate every tenant's credentials, write availability, or cross-region availability.
+
+The deployment must provide a dedicated canary tenant and credentials for this request and must
+provision a `datastax_sla.check` table that the canary principal can read. Its replication factor
+must be appropriate for the deployment (greater than one in a multi-node local data center) so
+`LOCAL_QUORUM` requires responses from multiple replicas. Astra callers must use the canary database
+hostname so the tenant and region are resolved from `Host`; Cassandra ignores the tenant portion of
+`Host`. The caller must also send the full User-Agent configured by
+`stargate.jsonapi.operations.sla-user-agent`. The comparison is case-insensitive. Requests with a
+missing or different User-Agent are rejected before accessing the session cache, and the endpoint
+fails closed when the SLA User-Agent is not configured. This ensures the canary session uses the
+shorter SLA session TTL instead of being treated like normal client traffic. Do not reuse the canary
+credentials for normal traffic, because using the same cached session with a non-SLA User-Agent can
+extend its lifetime.
+
+The check is fully asynchronous and has a five-second timeout. It returns HTTP 200 with
+`{"status":"UP"}` after a successful read and HTTP 503 with `{"status":"DOWN"}` after a database
+failure, timeout, or missing SLA User-Agent configuration. It returns the standard Data API error
+response with HTTP 401 when the `Token` header is missing or authentication fails, and HTTP 403 when
+the request User-Agent does not match the configured SLA User-Agent. Probe integrations must use the
+HTTP status as the readiness contract rather than parsing the response body's `status` field alone.
+
+Kubernetes or an SLA checker must call each pod directly for this endpoint to control per-pod
+readiness. An external request sent through a load balancer does not establish which pod is ready.
+Restrict the endpoint to trusted probe traffic with deployment controls such as a NetworkPolicy,
+mTLS, or an ingress ACL and rate limit. The User-Agent check is an operational guard, not an
+authentication boundary. Kubernetes `httpGet` headers cannot reference a Secret, so
+delivery of the canary token is intentionally outside the Data API configuration. Prefer an
+external checker or a Secret-mounted file read by an `exec` probe; do not put the token literally in
+the probe command or shell trace. The unauthenticated Quarkus health endpoints under the
+non-application path do not include this database check.
+
 
 ## Jsonapi metering configuration
 *Configuration for jsonapi metering, defined by [JsonApiMetricsConfig.java](io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java).*

src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java

@@ -0,0 +1,73 @@
+package io.stargate.sgv2.jsonapi.api.health;
+
+import com.datastax.oss.driver.api.core.DefaultConsistencyLevel;
+import com.datastax.oss.driver.api.core.cql.SimpleStatement;
+import com.google.common.annotations.VisibleForTesting;
+import io.smallrye.mutiny.Uni;
+import io.stargate.sgv2.jsonapi.api.request.RequestContext;
+import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache;
+import io.stargate.sgv2.jsonapi.service.cqldriver.executor.CommandQueryExecutor;
+import java.time.Duration;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * Runs the database probe exposed at {@code GET /v1/health/ready}.
+ *
+ * <p>This class is constructed by the JAX-RS resource and is not a CDI bean or a MicroProfile
+ * health check. The caller's request context supplies the tenant, token, and User-Agent for both
+ * Astra and Cassandra connections.
+ */
+public final class DatabaseReadinessCheck {
+
+  private static final String READINESS_QUERY = "SELECT * FROM datastax_sla.check LIMIT 1";
+  private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5);
+
+  private final Supplier<CQLSessionCache> sessionCacheSupplier;
+  private final SimpleStatement statement;
+  private final Duration timeout;
+
+  public DatabaseReadinessCheck(Supplier<CQLSessionCache> sessionCacheSupplier) {
+    this(sessionCacheSupplier, DEFAULT_TIMEOUT);
+  }
+
+  @VisibleForTesting
+  DatabaseReadinessCheck(CQLSessionCache sessionCache, Duration timeout) {
+    this(() -> sessionCache, timeout);
+  }
+
+  private DatabaseReadinessCheck(Supplier<CQLSessionCache> sessionCacheSupplier, Duration timeout) {
+    this.sessionCacheSupplier =
+        Objects.requireNonNull(sessionCacheSupplier, "sessionCacheSupplier must not be null");
+    this.timeout = Objects.requireNonNull(timeout, "timeout must not be null");
+    this.statement =
+        SimpleStatement.builder(READINESS_QUERY)
+            .setConsistencyLevel(DefaultConsistencyLevel.LOCAL_QUORUM)
+            .setTimeout(timeout)
+            .build();
+  }
+
+  /**
+   * Executes a replicated table read at {@code LOCAL_QUORUM}, using the {@code table-read} driver
+   * profile for the remaining read settings.
+   */
+  public Uni<Void> check(RequestContext requestContext) {
+    Objects.requireNonNull(requestContext, "requestContext must not be null");
+
+    return Uni.createFrom()
+        .deferred(
+            () -> {
+              var sessionCache =
+                  Objects.requireNonNull(
+                      sessionCacheSupplier.get(), "sessionCacheSupplier returned null");
+              return new CommandQueryExecutor(
+                      sessionCache, requestContext, CommandQueryExecutor.QueryTarget.TABLE)
+                  .executeRead(statement)
+                  .replaceWithVoid();
+            })
+        // The statement timeout bounds driver I/O; this also bounds asynchronous session lookup.
+        .ifNoItem()
+        .after(timeout)
+        .fail();
+  }
+}

src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java

@@ -0,0 +1,139 @@
+package io.stargate.sgv2.jsonapi.api.v1;
+
+import io.quarkus.security.UnauthorizedException;
+import io.smallrye.mutiny.Uni;
+import io.stargate.sgv2.jsonapi.api.health.DatabaseReadinessCheck;
+import io.stargate.sgv2.jsonapi.api.model.command.CommandResult;
+import io.stargate.sgv2.jsonapi.api.model.command.tracing.RequestTracing;
+import io.stargate.sgv2.jsonapi.api.request.RequestContext;
+import io.stargate.sgv2.jsonapi.config.constants.OpenApiConstants;
+import io.stargate.sgv2.jsonapi.exception.APISecurityException;
+import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import org.eclipse.microprofile.openapi.annotations.Operation;
+import org.eclipse.microprofile.openapi.annotations.media.Content;
+import org.eclipse.microprofile.openapi.annotations.media.Schema;
+import org.eclipse.microprofile.openapi.annotations.responses.APIResponse;
+import org.eclipse.microprofile.openapi.annotations.responses.APIResponses;
+import org.eclipse.microprofile.openapi.annotations.security.SecurityRequirement;
+import org.jboss.resteasy.reactive.RestResponse;
+
+/**
+ * Authenticated database readiness endpoint registered through Quarkus JAX-RS resource discovery.
+ *
+ * <p>{@code GET /v1/health/ready} runs the same request-scoped probe for Astra and Cassandra. A
+ * request must use the configured SLA User-Agent. A successful probe returns HTTP 200; invalid
+ * credentials return HTTP 401; a missing or different SLA User-Agent returns HTTP 403; and a
+ * database failure, timeout, or missing SLA configuration returns HTTP 503. The existing {@code
+ * /v1/*} security policy rejects requests without a token before this resource is called.
+ */
+@Path(DatabaseReadinessResource.BASE_PATH)
+@Produces(MediaType.APPLICATION_JSON)
+@SecurityRequirement(name = OpenApiConstants.SecuritySchemes.TOKEN)
+public class DatabaseReadinessResource {
+
+  public static final String BASE_PATH = GeneralResource.BASE_PATH + "/health/ready";
+
+  private static final ReadinessResponse UP = new ReadinessResponse("UP");
+  private static final ReadinessResponse DOWN = new ReadinessResponse("DOWN");
+
+  private final DatabaseReadinessCheck readinessCheck;
+  private final RequestContext requestContext;
+  private final CqlSessionCacheSupplier sessionCacheSupplier;
+
+  @Inject
+  public DatabaseReadinessResource(
+      CqlSessionCacheSupplier sessionCacheSupplier, RequestContext requestContext) {
+    this.readinessCheck = new DatabaseReadinessCheck(sessionCacheSupplier);
+    this.requestContext = requestContext;
+    this.sessionCacheSupplier = sessionCacheSupplier;
+  }
+
+  @GET
+  @Operation(
+      summary = "Check database readiness",
+      description =
+          "Uses the authenticated request tenant and token to perform a LOCAL_QUORUM read.")
+  @APIResponses({
+    @APIResponse(
+        responseCode = "200",
+        description = "The database completed the readiness read.",
+        content =
+            @Content(
+                mediaType = MediaType.APPLICATION_JSON,
+                schema = @Schema(implementation = ReadinessResponse.class))),
+    @APIResponse(
+        responseCode = "401",
+        description = "The token is missing or invalid.",
+        content =
+            @Content(
+                mediaType = MediaType.APPLICATION_JSON,
+                schema = @Schema(implementation = CommandResult.class))),
+    @APIResponse(
+        responseCode = "403",
+        description = "The request does not use the configured SLA User-Agent."),
+    @APIResponse(
+        responseCode = "503",
+        description = "The SLA User-Agent is not configured, or the database check failed.",
+        content =
+            @Content(
+                mediaType = MediaType.APPLICATION_JSON,
+                schema = @Schema(implementation = ReadinessResponse.class)))
+  })
+  public Uni<RestResponse<Object>> ready() {
+    var configuredSlaUserAgent = sessionCacheSupplier.slaUserAgent();
+    if (configuredSlaUserAgent.isEmpty()) {
+      return Uni.createFrom().item(response(Response.Status.SERVICE_UNAVAILABLE, DOWN));
+    }
+    if (!configuredSlaUserAgent.get().equals(requestContext.userAgent())) {
+      return Uni.createFrom().item(response(Response.Status.FORBIDDEN));
+    }
+
+    return readinessCheck
+        .check(requestContext)
+        .map(ignored -> response(Response.Status.OK, UP))
+        .onFailure(DatabaseReadinessResource::isUnauthorized)
+        .recoverWithItem(failure -> unauthorizedResponse())
+        .onFailure()
+        .recoverWithItem(failure -> response(Response.Status.SERVICE_UNAVAILABLE, DOWN));
+  }
+
+  private static boolean isUnauthorized(Throwable failure) {
+    var current = failure;
+    var seen = Collections.newSetFromMap(new IdentityHashMap<Throwable, Boolean>());
+    while (current != null && seen.add(current)) {
+      if (current instanceof UnauthorizedException
+          || current instanceof APISecurityException apiException
+              && apiException.httpStatus == Response.Status.UNAUTHORIZED.getStatusCode()) {
+        return true;
+      }
+      current = current.getCause();
+    }
+    return false;
+  }
+
+  private static RestResponse<Object> unauthorizedResponse() {
+    var commandResult =
+        CommandResult.statusOnlyBuilder(RequestTracing.NO_OP)
+            .addThrowable(APISecurityException.Code.UNAUTHENTICATED_REQUEST.get())
+            .build();
+    return response(Response.Status.UNAUTHORIZED, commandResult);
+  }
+
+  private static RestResponse<Object> response(Response.Status status) {
+    return RestResponse.ResponseBuilder.<Object>create(status).build();
+  }
+
+  private static RestResponse<Object> response(Response.Status status, Object entity) {
+    return RestResponse.ResponseBuilder.<Object>create(status, entity).build();
+  }
+
+  public record ReadinessResponse(String status) {}
+}

src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java

@@ -21,6 +21,7 @@
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Tags;
 import io.stargate.sgv2.jsonapi.api.request.RequestContext;
+import io.stargate.sgv2.jsonapi.api.v1.DatabaseReadinessResource;
 import io.stargate.sgv2.jsonapi.api.v1.metrics.MetricsConfig;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
@@ -71,31 +72,37 @@ public TenantRequestMetricsFilter(
   @ServerResponseFilter
   public void record(
       ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
-    // only if enabled
-    if (config.enabled()) {
-
-      // resolve tenant
-      Tag tenantTag = Tag.of(config.tenantTag(), this.requestContext.tenant().toString());
+    if (!config.enabled() || isDatabaseReadinessRequest(requestContext)) {
+      return;
+    }
 
-      // resolve error
-      boolean error = responseContext.getStatus() >= 500;
-      Tag errorTag = error ? ExceptionMetrics.TAG_ERROR_TRUE : ExceptionMetrics.TAG_ERROR_FALSE;
+    // resolve tenant
+    Tag tenantTag = Tag.of(config.tenantTag(), this.requestContext.tenant().toString());
 
-      // check if we need user agent as well
-      Tags tags = Tags.of(tenantTag, errorTag);
-      if (config.userAgentTagEnabled()) {
-        String userAgentValue = getUserAgentValue(requestContext);
-        tags = tags.and(Tag.of(config.userAgentTag(), userAgentValue));
-      }
+    // resolve error
+    boolean error = responseContext.getStatus() >= 500;
+    Tag errorTag = error ? ExceptionMetrics.TAG_ERROR_TRUE : ExceptionMetrics.TAG_ERROR_FALSE;
 
-      // add http status code
-      if (config.statusTagEnabled()) {
-        tags = tags.and(Tag.of(config.statusTag(), String.valueOf(responseContext.getStatus())));
-      }
+    // check if we need user agent as well
+    Tags tags = Tags.of(tenantTag, errorTag);
+    if (config.userAgentTagEnabled()) {
+      String userAgentValue = getUserAgentValue(requestContext);
+      tags = tags.and(Tag.of(config.userAgentTag(), userAgentValue));
+    }
 
-      // record
-      meterRegistry.counter(config.metricName(), tags).increment();
+    // add http status code
+    if (config.statusTagEnabled()) {
+      tags = tags.and(Tag.of(config.statusTag(), String.valueOf(responseContext.getStatus())));
     }
+
+    // record
+    meterRegistry.counter(config.metricName(), tags).increment();
+  }
+
+  private static boolean isDatabaseReadinessRequest(ContainerRequestContext requestContext) {
+    var requestPath = requestContext.getUriInfo().getRequestUri().getPath();
+    return DatabaseReadinessResource.BASE_PATH.equals(requestPath)
+        || (DatabaseReadinessResource.BASE_PATH + "/").equals(requestPath);
   }
 
   private String getUserAgentValue(ContainerRequestContext requestContext) {

src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplier.java

@@ -10,6 +10,7 @@
 import java.time.Duration;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
@@ -23,6 +24,7 @@
 public class CqlSessionCacheSupplier implements Supplier<CQLSessionCache> {
 
   private final CQLSessionCache singleton;
+  private final Optional<UserAgent> slaUserAgent;
 
   @Inject
   public CqlSessionCacheSupplier(
@@ -53,11 +55,14 @@ public CqlSessionCacheSupplier(
             dbConfig.cassandraPort(),
             () -> schemaObjectCacheSupplier.get().getSchemaChangeListener());
 
+    slaUserAgent =
+        operationsConfig.slaUserAgent().filter(value -> !value.isBlank()).map(UserAgent::new);
+
     singleton =
         new CQLSessionCache(
             dbConfig.sessionCacheMaxSize(),
             Duration.ofSeconds(dbConfig.sessionCacheTtlSeconds()),
-            operationsConfig.slaUserAgent().map(UserAgent::new).orElse(null),
+            slaUserAgent.orElse(null),
             Duration.ofSeconds(dbConfig.slaSessionCacheTtlSeconds()),
             credentialsFactory,
             sessionFactory,
@@ -70,4 +75,9 @@ public CqlSessionCacheSupplier(
   public CQLSessionCache get() {
     return singleton;
   }
+
+  /** Gets the configured User-Agent that selects the shorter SLA session-cache TTL. */
+  public Optional<UserAgent> slaUserAgent() {
+    return slaUserAgent;
+  }
 }

src/main/resources/application.yaml

@@ -159,7 +159,7 @@ quarkus:
       http-server:
         # ignore all non-application uris, as well as the custom set
         suppress-non-application-uris: true
-        ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html
+        ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html,/v1/health/ready/?
 
         # due to the https://github.com/quarkusio/quarkus/issues/24938
         # we need to define uri templating on our own for now