diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index beccd0a9..12e6b743 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -56,6 +56,7 @@ import app.softnetwork.elastic.sql.query.{ DropWatcher, EnrichPolicyStatement, ExecuteEnrichPolicy, + FromlessSelect, Insert, LicenseStatement, MultiSearch, @@ -541,16 +542,23 @@ class TableExecutor( Future.successful( ElasticResult.success( QueryRows( - mappings.map { case (index, mappings) => - ListMap( - "name" -> index, - "type" -> mappings.tableType.name.toUpperCase, - "pk" -> mappings.primaryKey.mkString(","), - "partitioned" -> mappings.partitionBy - .map(p => s"PARTITION BY ${p.column} (${p.granularity})") - .getOrElse("") - ) - }.toSeq + mappings + // Issue #251/AD-10 — the handshake index is infrastructure, not a table: + // never listed, ANY pattern. This one seam also covers jdbc getTables and + // Flight GET_TABLES (both execute SHOW TABLES through gateway.run). + // DESCRIBE TABLE deliberately still works on it. + .filterNot { case (index, _) => index == GatewayApi.HandshakeIndex } + .map { case (index, mappings) => + ListMap( + "name" -> index, + "type" -> mappings.tableType.name.toUpperCase, + "pk" -> mappings.primaryKey.mkString(","), + "partitioned" -> mappings.partitionBy + .map(p => s"PARTITION BY ${p.column} (${p.granularity})") + .getOrElse("") + ) + } + .toSeq ) ) ) @@ -1677,6 +1685,40 @@ class LicenseExecutor( exp.map(_.toString).getOrElse("never") } +/** Issue #251 (story 20.9) — FROM-less SELECT: Painless handshake AGAINST Elasticsearch. The + * statement's LIMIT/OFFSET are applied engine-side on the one assembled row, AFTER the ES + * round-trip (AD-12) — `LIMIT 0` still connection-checks (AD-8), and the internal rewrite always + * carries its own `LIMIT 1` so it can never reach scroll/PIT. + */ +class FromlessSelectExecutor( + evaluator: HandshakeEvaluator, + logger: Logger +) extends Executor[FromlessSelect] { + + override def execute( + statement: FromlessSelect + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + // run(statement) never calls validate(): a programmatic FromlessSelect reaches this + // executor without the parser's gate — re-run the guards (surviving review critical #5). + statement.validate() match { + case Left(reason) => + val error = + ElasticError(message = reason, statusCode = Some(400), operation = Some("sql")) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + case Right(_) => + evaluator.evaluateHandshake(statement).map { + case ElasticSuccess(row) => + val offset = statement.limit.flatMap(_.offset).map(_.offset).getOrElse(0) + val max = statement.limit.map(_.limit).getOrElse(1) + ElasticSuccess(QueryRows(Seq(row).drop(offset).take(max))) + case ElasticFailure(error) => ElasticFailure(error) + } + } + } +} + class DqlRouterExecutor( searchExec: SearchExecutor, pipelineExec: PipelineExecutor, @@ -1684,7 +1726,8 @@ class DqlRouterExecutor( watcherExec: WatcherExecutor, policyExec: EnrichPolicyExecutor, clusterExec: ClusterExecutor, - licenseExec: LicenseExecutor + licenseExec: LicenseExecutor, + fromlessExec: FromlessSelectExecutor // issue #251 — see the story 20.9 AD-4′ arity note ) extends Executor[DqlStatement] { override def execute( @@ -1698,6 +1741,8 @@ class DqlRouterExecutor( case e: EnrichPolicyStatement => policyExec.execute(e) case c: ClusterStatement => clusterExec.execute(c) case l: LicenseStatement => licenseExec.execute(l) + // Issue #251 — FROM-less SELECT: Painless handshake AGAINST Elasticsearch. + case f: FromlessSelect => fromlessExec.execute(f) case _ => Future.successful( @@ -1775,6 +1820,18 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers { strategy = licenseRefreshStrategy // No longer Option — NopRefreshStrategy for Community ) + /** Issue #251 — the seam behind the FROM-less SELECT handshake. Today's backend searches the + * dedicated handshake index; a future Painless-execute-API backend swaps in HERE, touching + * nothing else (AD-4′). + */ + lazy val handshakeEvaluator: HandshakeEvaluator = + new SearchHandshakeEvaluator(api = this, logger = logger) + + lazy val fromlessSelectExecutor = new FromlessSelectExecutor( + evaluator = handshakeEvaluator, + logger = logger + ) + lazy val dqlExecutor = new DqlRouterExecutor( searchExec = searchExecutor, pipelineExec = pipelineExecutor, @@ -1782,7 +1839,8 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers { watcherExec = watcherExecutor, policyExec = policyExecutor, clusterExec = clusterExecutor, - licenseExec = licenseExecutor + licenseExec = licenseExecutor, + fromlessExec = fromlessSelectExecutor ) lazy val ddlExecutor = new DdlRouterExecutor( @@ -2075,6 +2133,27 @@ object GatewayApi { s"$ParseRejectionPrefix [${excerpt(statement)}]: $safeReason" } + /** Issue #251 — the dedicated FROM-less-handshake index. NOT dot-prefixed (ES 8+ + * deprecation-warns dot-prefixed non-system creation — and the deprecation LOG itself creates a + * `.ds-*` index, the measured all_templates trap); product-prefixed so operators can attribute + * it; excluded from SHOW TABLES by TableExecutor (AD-10). + */ + val HandshakeIndex: String = "softclient4es_handshake" + + /** 1 shard / 0 replicas: single-node clusters stay green. NEVER defaultSettings (ngram). */ + private[client] val HandshakeSettings: String = + """{"index": {"number_of_shards": 1, "number_of_replicas": 0}}""" + + /** index.hidden exists only from ES 7.7 — used when the cluster supports it (AD-9/AD-10). */ + private[client] val HandshakeSettingsHidden: String = + """{"index": {"number_of_shards": 1, "number_of_replicas": 0, "hidden": true}}""" + + private[client] val HandshakeMapping: String = + """{"properties": {"dummy": {"type": "keyword"}}}""" + + private[client] val HandshakeDocId: String = "1" + private[client] val HandshakeDoc: String = """{"dummy": "dummy"}""" + /** Split a normalized SQL string into statements on top-level `;`. * * This replaces `split(";\\s*$")`, which — `$` anchoring to the end of the whole (newline-free) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/HandshakeEvaluator.scala b/core/src/main/scala/app/softnetwork/elastic/client/HandshakeEvaluator.scala new file mode 100644 index 00000000..8f152f82 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/HandshakeEvaluator.scala @@ -0,0 +1,237 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.query.{FromlessSelect, SingleSearch} +import org.slf4j.Logger + +import scala.collection.immutable.ListMap +import scala.concurrent.{ExecutionContext, Future} +import scala.util.Try + +/** THE SEAM (issue #251, AD-4′): "evaluate this FROM-less select-list's Painless against the + * cluster and give me ONE row". Today's backend searches the dedicated handshake index + * (script_fields on the seeded doc); the recorded future backend is the Painless execute API + * (`_scripts/painless/_execute`, no index needed) — swapping it must touch nothing but this + * trait's implementation wiring in GatewayApi. + */ +trait HandshakeEvaluator { + def evaluateHandshake(statement: FromlessSelect)(implicit + system: ActorSystem + ): Future[ElasticResult[ListMap[String, Any]]] +} + +/** Search-backed implementation: lazily ensures the handshake index (probe-before-act, race-safe, + * memoized — AD-9), rewrites the statement to `SELECT FROM LIMIT 1` (AD-3′) + * and executes it through the UNMODIFIED FROM-ful pipeline, then assembles one row: script_fields + * arrays unwrapped, `__cN` keys renamed to the PD-2 output names (AD-11). + */ +class SearchHandshakeEvaluator( + api: SearchApi with IndicesApi with IndexApi with RefreshApi with VersionApi, + logger: Logger +) extends HandshakeEvaluator { + + import GatewayApi._ + + @volatile private[this] var ready: Boolean = false + + /** Test seam only. */ + private[client] def isReady: Boolean = ready + + override def evaluateHandshake(statement: FromlessSelect)(implicit + system: ActorSystem + ): Future[ElasticResult[ListMap[String, Any]]] = { + implicit val ec: ExecutionContext = system.dispatcher + implicit val context: ConversionContext = NativeContext + ensureHandshakeIndex() match { + case ElasticFailure(error) => Future.successful(ElasticFailure(error)) + case ElasticSuccess(_) => runSearch(statement, retriesLeft = 1) + } + } + + private def runSearch(statement: FromlessSelect, retriesLeft: Int)(implicit + ec: ExecutionContext, + context: ConversionContext + ): Future[ElasticResult[ListMap[String, Any]]] = { + val single = statement.toSingleSearch(HandshakeIndex) + api.searchAsync(single).flatMap { + case ElasticSuccess(response) => + response.results.headOption match { + case Some(raw) => + Future.successful(ElasticSuccess(assembleRow(statement, single, raw))) + case None if retriesLeft > 0 => + // Pre-created-but-unseeded index, or the seed doc was deleted out-of-band: + // re-seed ONCE, then loud (never an empty-but-successful answer — #253 family). + ready = false + ensureHandshakeIndex(forceSeed = true) match { + case ElasticFailure(error) => Future.successful(ElasticFailure(error)) + case _ => runSearch(statement, retriesLeft - 1) + } + case None => + Future.successful(ElasticFailure(handshakeCorruptError())) + } + case ElasticFailure(error) if indexNotFound(error) && retriesLeft > 0 => + // Out-of-band index delete: reset the memo, re-ensure, retry ONCE. + ready = false + ensureHandshakeIndex() match { + case ElasticFailure(e) => Future.successful(ElasticFailure(e)) + case _ => runSearch(statement, retriesLeft - 1) + } + case ElasticFailure(error) => + // The connection check doing its job: propagate the client/cluster failure verbatim. + Future.successful(ElasticFailure(error)) + } + } + + /** Probe-before-act (project_mv_metadata_index_contract): an ElasticFailure from the probe is + * NEVER read as "absent". A failed create re-probes ONCE (lost cross-process race => proceed) + * and otherwise propagates the ORIGINAL failure (a 403 stays a 403 — PD-6/OQ-6). Memoized per + * client; concurrent ensures are idempotent (PUT same doc id). Runs synchronously on the caller + * thread — the established extension-path shape, once per client lifecycle. + */ + private[client] def ensureHandshakeIndex(forceSeed: Boolean = false): ElasticResult[Unit] = { + if (ready && !forceSeed) ElasticResult.success(()) + else { + api.indexExists(HandshakeIndex, pattern = false) match { + case ElasticFailure(error) if error.statusCode.contains(403) => + // A read-only account can be denied the EXISTS probe itself (ES security answers the + // exists action with 403 for an index the user has no privilege on) — the OQ-6/PD-6 + // guidance must reach THIS failure too, not only the create/seed 403s, or exactly the + // read-only BI session the guidance exists for gets a bare security_exception. + withGuidance(ElasticFailure(error)) + case ElasticFailure(error) => + ElasticFailure(error) // outage != absence — propagate verbatim (never "absent") + case ElasticSuccess(true) if !forceSeed => + ready = true + ElasticResult.success(()) + case ElasticSuccess(existsNow) => + val created: ElasticResult[_] = + if (existsNow) ElasticResult.success(true) + else + // mappings MUST be passed: without it the seed doc dynamic-maps `dummy` as + // text+keyword instead of the lead-mandated single keyword field (AC 5/AD-9), + // and the HandshakeMapping constant is dead code. + api.createIndex( + HandshakeIndex, + settings = handshakeSettings(), + mappings = Some(HandshakeMapping) + ) match { + case f @ ElasticFailure(_) => + api.indexExists(HandshakeIndex, pattern = false) match { + case ElasticSuccess(true) => ElasticResult.success(true) // lost the race + case _ => withGuidance(f) + } + case ok => ok + } + created match { + case ElasticFailure(error) => ElasticFailure(error) + case _ => + api.index(HandshakeIndex, HandshakeDocId, HandshakeDoc) match { + case ElasticFailure(error) => withGuidance(ElasticFailure(error)) + case _ => + api.refresh(HandshakeIndex) match { + case ElasticFailure(error) => ElasticFailure(error) + case _ => + ready = true + logger.info(s"✅ Handshake index '$HandshakeIndex' ready") + ElasticResult.success(()) + } + } + } + } + } + } + + /** index.hidden exists only from ES 7.7 (AD-9); `api.version` caches successes. A version lookup + * failure — or an unparseable version string — falls back to the un-hidden settings (the create + * itself will surface any real outage) — defense-in-depth must not add a failure mode. + */ + private def handshakeSettings(): String = + api.version match { + case ElasticSuccess(v) if Try(ElasticsearchVersion.isAtLeast(v, 7, 7)).getOrElse(false) => + HandshakeSettingsHidden + case _ => HandshakeSettings + } + + /** Bounded self-heal trigger. statusCode None != 404 (project_elastic_error_status_semantics) — + * the message probe covers status-less transports; a wrong trigger costs one retry, never a + * wrong answer. + */ + private def indexNotFound(error: ElasticError): Boolean = + error.statusCode.contains(404) || + Option(error.message).exists(_.contains("index_not_found")) + + private def handshakeCorruptError(): ElasticError = + ElasticError( + message = + s"FROM-less SELECT handshake found index '$HandshakeIndex' present but empty and could " + + s"not re-seed it. Seed it once: PUT /$HandshakeIndex/_doc/$HandshakeDocId $HandshakeDoc", + statusCode = Some(500), + index = Some(HandshakeIndex), + operation = Some("handshake") + ) + + /** PD-6/OQ-6 recommendation (lead-confirmed default): keep the original failure — status + * included, never invented — and append the pre-creation guidance for read-only BI service + * accounts. Both routes are named (lead review of PR #268): the SQL one for an administrator + * connected through SoftClient4ES itself, the REST one for curl/Kibana. + */ + private def withGuidance(f: ElasticFailure): ElasticFailure = + ElasticFailure( + f.elasticError.copy( + message = s"${f.elasticError.message} — FROM-less SELECT executes a Painless handshake " + + s"against index '$HandshakeIndex'. If this client must stay read-only, create it " + + s"once as an administrator — via SQL: CREATE TABLE IF NOT EXISTS $HandshakeIndex " + + s"""(dummy KEYWORD) OPTIONS (settings = (number_of_shards = "1", """ + + s"""number_of_replicas = "0")); INSERT INTO $HandshakeIndex (dummy) VALUES ('dummy'); """ + + s"or via REST: PUT /$HandshakeIndex " + + s"""{"settings": {"number_of_shards": 1, "number_of_replicas": 0}, """ + + s""""mappings": $HandshakeMapping} then PUT /$HandshakeIndex/_doc/$HandshakeDocId """ + + s"$HandshakeDoc — the driver then uses it read-only.", + index = Some(HandshakeIndex), + operation = Some("handshake") + ) + ) + + /** Project EXACTLY the select-list outputs (PD-2 names), unwrapping the ES per-field + * script_fields array (AD-11): the generic parseSimpleHits row keeps the wrap AND appends the + * `dummy` _source entry (normalizeRow "extra fields"). Response keys = the rewrite's computed + * aliases (`__cN`) or explicit aliases — positionally zipped with the statement's output names + * (same Select instance, same order; key alignment is by construction: SingleSearch.scriptFields + * = fieldsWithComputedAliases.filter(_.isScriptField)). + */ + private def assembleRow( + statement: FromlessSelect, + single: SingleSearch, + raw: ListMap[String, Any] + ): ListMap[String, Any] = { + val responseKeys = + single.select.fieldsWithComputedAliases.map(f => + f.fieldAlias.map(_.alias).getOrElse(f.sourceField) + ) + ListMap(statement.columnNames.zip(responseKeys).map { case (out, key) => + out -> (raw.get(key) match { + case Some(l: Seq[_]) if l.size == 1 => l.head // the ES per-field array wrapper + case Some(l: Seq[_]) if l.isEmpty => null + case Some(v) => v // defensive passthrough — never guess + case None => null // script returned null -> key absent + }) + }: _*) + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala new file mode 100644 index 00000000..86b50466 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala @@ -0,0 +1,240 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.immutable.ListMap +import scala.collection.mutable +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} + +/** Story 20.9 / issue #251 — FROM-less SELECT handshake, Docker-free half. + * + * Covers routing + row assembly + connection-check semantics on `NopeClientApi`-derived fixtures. + * Value truth lives in the testkit integration specs (semantics execute on real ES); here the + * stubs RECORD and the tests ASSERT — never a matcher inside a stub. + */ +class FromlessSelectGatewaySpec + extends AnyFlatSpec + with Matchers + with ScalaFutures + with BeforeAndAfterAll { + + implicit private val system: ActorSystem = ActorSystem("fromless-select-gateway") + override implicit val patienceConfig: PatienceConfig = + PatienceConfig(timeout = scaled(5.seconds)) + + private val mapper = new ObjectMapper() + + private val minimalMappingJson = """{"properties": {"name": {"type": "keyword"}}}""" + + override def afterAll(): Unit = { + system.terminate() + super.afterAll() + } + + private def rows(result: ElasticResult[QueryResult]): Seq[ListMap[String, Any]] = + result match { + case ElasticSuccess(QueryRows(r, _)) => r + case other => fail(s"expected QueryRows, got $other") + } + + // ── Fixture B: a NopeClientApi whose search answers from a response QUEUE and which + // RECORDS index-admin calls. NopeClientApi returns benign no-op successes by default + // (NOT throws — corrected fact), so only the handshake-relevant members are overridden. + private class StubHandshakeClient extends NopeClientApi { + override protected def logger: Logger = LoggerFactory.getLogger(getClass) + + val responses: mutable.Queue[String] = mutable.Queue.empty + val created: mutable.Buffer[String] = mutable.Buffer.empty + var createdMappings: Option[String] = None // pins AD-9: the keyword mapping MUST be passed + val seeded: mutable.Buffer[(String, String, String)] = mutable.Buffer.empty + val refreshed: mutable.Buffer[String] = mutable.Buffer.empty + var existing: Boolean = false + + def respond(fieldsJson: String): Unit = + responses.enqueue( + s"""{"hits":{"total":{"value":1},"hits":[{"_index":"x","_id":"1", + |"_source":{"dummy":"dummy"},"fields":$fieldsJson}]}}""".stripMargin + ) + + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = + ElasticResult.success(existing) + override private[client] def executeCreateIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[app.softnetwork.elastic.sql.schema.TableAlias] + ): ElasticResult[Boolean] = { + created += index; createdMappings = mappings; existing = true; ElasticResult.success(true) + } + override private[client] def executeIndex( + index: String, + id: String, + source: String, + wait: Boolean + ): ElasticResult[Boolean] = { seeded += ((index, id, source)); ElasticResult.success(true) } + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = { + refreshed += index; ElasticResult.success(true) + } + override private[client] def executeSingleSearchAsync(elasticQuery: ElasticQuery)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[JsonNode]]] = + Future.successful( + if (responses.isEmpty) ElasticResult.success(None) + else ElasticResult.success(Some(mapper.readTree(responses.dequeue()))) + ) + } + + // ── (A) nothing answers locally: bare NopeClientApi => the handshake FAILS (AC 4) ── + "GatewayApi.run(SELECT 1)" should "never be answered engine-side (bare no-op client fails)" in { + val client = new NopeClientApi { + override protected def logger: Logger = LoggerFactory.getLogger(getClass) + } + client.run("SELECT 1").futureValue match { + case ElasticFailure(_) => succeed // ensure or search failed — no local answer exists + case other => fail(s"a FROM-less SELECT must not succeed without ES: $other") + } + } + + // ── (B) row assembly, unwrap, naming, LIMIT semantics (AC 1, 7) ──────────── + it should "assemble one unwrapped row under PD-2 names" in { + val client = new StubHandshakeClient + client.respond("""{"__c1":[1]}""") + rows(client.run("SELECT 1").futureValue) shouldBe Seq(ListMap("1" -> 1)) + // ensure-flow ran exactly once and in order, with the keyword mapping (AD-9 — not dynamic). + // The mapping reaches executeCreateIndex CONVERTED per ES version (MappingConverter may add + // the ES-6 `_doc` type wrapper), so pin presence + content, not byte equality. + client.created shouldBe Seq(GatewayApi.HandshakeIndex) + client.createdMappings should not be empty + client.createdMappings.get should include(""""dummy"""") + client.createdMappings.get should include(""""keyword"""") + client.seeded.map(_._2) shouldBe Seq(GatewayApi.HandshakeDocId) + client.refreshed shouldBe Seq(GatewayApi.HandshakeIndex) + + client.respond("""{"x":[1],"s":["ok"]}""") + rows(client.run("SELECT 1 AS x, 'ok' AS s").futureValue) shouldBe + Seq(ListMap("x" -> 1, "s" -> "ok")) + // memoized: no further create/seed/refresh + client.created.size shouldBe 1 + } + + it should "honour LIMIT engine-side while still executing the round-trip" in { + val client = new StubHandshakeClient + client.respond("""{"__c1":[1]}""") + rows(client.run("SELECT 1 LIMIT 100").futureValue).size shouldBe 1 + client.respond("""{"__c1":[1]}""") + rows(client.run("SELECT 1 LIMIT 0").futureValue) shouldBe Seq.empty + client.responses.isEmpty shouldBe true // LIMIT 0 still consumed a search (AD-8) + } + + it should "run a multi-statement handshake batch and return the last result" in { + val client = new StubHandshakeClient + client.respond("""{"__c1":[1]}"""); client.respond("""{"__c1":[2]}""") + rows(client.run("SELECT 1; SELECT 2").futureValue) shouldBe Seq(ListMap("2" -> 2)) + } + + // ── (C) connection-check semantics (AC 4) ────────────────────────────────── + it should "fail with the propagated client error when the cluster is unreachable" in { + val client = new StubHandshakeClient { + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = + ElasticResult.failure( + ElasticError(message = "Connection refused: localhost/127.0.0.1:9200") + ) + } + client.run("SELECT 1").futureValue match { + case ElasticFailure(error) => + error.message should include("Connection refused") + // NOT pinned: statusCode (None is legitimate here — project_elastic_error_status_semantics) + case other => fail(s"expected connection failure, got $other") + } + } + + it should "append the pre-creation guidance when the exists probe itself is denied (403)" in { + // Read-only BI account, index never created: ES security can 403 the EXISTS action itself — + // the OQ-6 guidance must reach that failure too (original status preserved, never invented). + val client = new StubHandshakeClient { + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = + ElasticResult.failure( + ElasticError(message = "security_exception: action denied", statusCode = Some(403)) + ) + } + client.run("SELECT 1").futureValue match { + case ElasticFailure(error) => + error.statusCode shouldBe Some(403) // the ORIGINAL status — no invention + error.message should include("security_exception") + error.message should include(GatewayApi.HandshakeIndex) // the guidance names the index + case other => fail(s"expected guided 403, got $other") + } + } + + it should "append the pre-creation guidance when the create itself is denied (403)" in { + val client = new StubHandshakeClient { + override private[client] def executeCreateIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[app.softnetwork.elastic.sql.schema.TableAlias] + ): ElasticResult[Boolean] = + ElasticResult.failure( + ElasticError(message = "security_exception: create denied", statusCode = Some(403)) + ) + } + client.run("SELECT 1").futureValue match { + case ElasticFailure(error) => + error.statusCode shouldBe Some(403) + error.message should include("security_exception") + error.message should include(GatewayApi.HandshakeIndex) + case other => fail(s"expected guided 403, got $other") + } + } + + // ── rejection path (AC 3, 12) — 20.4-independent ─────────────────────────── + it should "reject column references with a 400 and a named reason" in { + val client = new StubHandshakeClient + client.run("SELECT col").futureValue match { + case ElasticFailure(error) => + error.statusCode shouldBe Some(400) + error.message should include("Column reference 'col' requires a FROM clause") + case other => fail(s"expected failure, got $other") + } + } + + // ── SHOW TABLES invisibility (AC 6), Docker-free half ────────────────────── + it should "never list the handshake index in SHOW TABLES" in { + val client = new StubHandshakeClient { + override private[client] def executeGetAllMappings( + indices: Seq[String] + ): ElasticResult[Map[String, String]] = + ElasticResult.success( + Map( + GatewayApi.HandshakeIndex -> minimalMappingJson, + "orders" -> minimalMappingJson + ) + ) + } + rows(client.run("SHOW TABLES").futureValue).map(_("name")) shouldBe Seq("orders") + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala index 17b9e942..51f7d05c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala @@ -25,7 +25,7 @@ import app.softnetwork.elastic.client.scroll.{ScrollConfig, ScrollMetrics} import app.softnetwork.elastic.licensing._ import app.softnetwork.elastic.licensing.metrics.MetricsApi import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{SearchStatement, SelectStatement, SingleSearch} +import app.softnetwork.elastic.sql.query.{Limit, SearchStatement, SelectStatement, SingleSearch} import com.typesafe.config.ConfigFactory import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -449,4 +449,43 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { res shouldBe a[ElasticSuccess[_]] capHits(collector).values.toSet shouldBe Set(0L) } + + // ---- Story 20.9 / issue #251: FROM-less SELECT takes the no-cap arm (AC 9) ---- + + behavior of "CoreDqlExtension FROM-less handshake routing (story 20.9)" + + it should "route a FromlessSelect down the structural no-cap arm — no quota, no cap-hit, never scroll" in { + // Pinned so a future cap refactor cannot silently start metering handshakes: FromlessSelect + // is neither SingleSearch nor SelectStatement, so checkQuotasAndExecute must fall through to + // `case _ => client.dqlExecutor.execute(dql)` — no cap applied, no CapHitKind.QueryResults + // increment. The INTERNAL rewrite then executes BELOW this seam as an ordinary SingleSearch + // carrying its own LIMIT 1 (never re-capped, never re-metered, can never reach scroll). + val parsed = Parser("SELECT 1") match { + case Right(s) => s + case Left(e) => fail(s"parse failed: ${e.msg}") + } + val collector = new TelemetryCollector + val client = new RecordingClient() + val ext = new CoreDqlExtension() + ext.initialize( + ConfigFactory.empty(), + strategy(managerWithQuota(Quota.Community, LicenseType.Community), collector) + ) + val res = Await.result(ext.execute(parsed, client), 5.seconds) + + res shouldBe a[ElasticSuccess[_]] // RecordingClient's canned searchAsync answers the rewrite + // never the scroll path — the rewrite's own LIMIT 1 keeps requiresScrollPaging false + client.scrolledStatement.get() shouldBe null + client.scrolledConfig.get() shouldBe null + // what reached the search seam is the INTERNAL rewrite: handshake index, LIMIT 1 — proof the + // statement crossed the extension seam uncapped and the rewrite was built below it + client.searchedStatement.get() match { + case s: SingleSearch => + s.from.tables.map(_.name) shouldBe Seq(GatewayApi.HandshakeIndex) + s.limit shouldBe Some(Limit(1, None)) + case other => fail(s"expected the handshake rewrite via searchAsync, got $other") + } + // no cap-hit of ANY kind + capHits(collector).values.toSet shouldBe Set(0L) + } } diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index 347de1c7..d27b139a 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -23,6 +23,7 @@ DQL supports: ## Table of Contents - [SELECT](#select) +- [FROM-less SELECT (connection handshake)](#from-less-select-connection-handshake) - [WHERE](#where) - [ORDER BY](#order-by) - [LIMIT / OFFSET](#limit--offset) @@ -85,6 +86,101 @@ ORDER BY id ASC; --- +## FROM-less SELECT (connection handshake) + +`SELECT` without a `FROM` clause is the connection/health idiom of the JDBC/SQLAlchemy +ecosystem: Tableau re-issues `SELECT 1` on every interaction, Superset's connection test sends +`SELECT 1`, its engine probe sends `SELECT 1 LIMIT 100`, and connection pools use it as +`connectionTestQuery`. All of these are supported: + +```sql +SELECT 1; +SELECT 1 AS x; +SELECT 1 LIMIT 100; +SELECT 1 AS ok, UPPER('x') AS u, 1+1 AS two; +SELECT CURRENT_TIMESTAMP AS ts; +SELECT '125'::BIGINT AS c; +``` + +Each returns **exactly one row**. Column names follow the usual convention: an explicit alias +wins, otherwise the rendered expression (`SELECT 1` yields a column named `1`). + +### Connection-check semantics + +A FROM-less `SELECT` **executes against the Elasticsearch cluster** — the select-list is +translated to Painless and evaluated by ES, exactly like the same expressions in a FROM-ful +query. Consequently, **with the cluster unreachable, `SELECT 1` fails** with the propagated +connection error. A green handshake genuinely means "connected"; a pool's +`connectionTestQuery = SELECT 1` is a real connection test. + +### The handshake index + +The first FROM-less `SELECT` on a client lazily creates a dedicated index in the cluster: + +- name: `softclient4es_handshake` +- settings: 1 shard, 0 replicas (single-node clusters stay green); `index.hidden: true` on + ES ≥ 7.7 +- mapping: a single `dummy` keyword field +- content: one seeded document (`PUT /softclient4es_handshake/_doc/1 {"dummy": "dummy"}`) + +Creation is race-safe and idempotent, and happens once per client lifecycle. The index is +**never listed by `SHOW TABLES`** (any pattern) — which also keeps it out of JDBC +`DatabaseMetaData.getTables` and Arrow Flight `GET_TABLES` browsing. `DESCRIBE TABLE +softclient4es_handshake` still works, deliberately, for debuggability. The index is never +deleted automatically. + +#### Read-only BI service accounts + +If the account the BI tool connects with cannot create indices, have an administrator +pre-create and seed the index once — lazy creation then becomes a no-op existence probe, and +the read-only account only needs the `read` privilege on `softclient4es_handshake`. + +Through SoftClient4ES itself (REPL, JDBC, or any connected client, with a privileged account): + +```sql +CREATE TABLE IF NOT EXISTS softclient4es_handshake (dummy KEYWORD) +OPTIONS (settings = (number_of_shards = "1", number_of_replicas = "0")); +INSERT INTO softclient4es_handshake (dummy) VALUES ('dummy'); +``` + +The `CREATE TABLE IF NOT EXISTS` is a no-op when the index already exists; re-running the +`INSERT` just adds another row, which is harmless — the handshake reads a single one. + +Or directly against Elasticsearch: + +``` +PUT /softclient4es_handshake +{ + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + "mappings": {"properties": {"dummy": {"type": "keyword"}}} +} + +PUT /softclient4es_handshake/_doc/1 +{"dummy": "dummy"} +``` + +Without pre-creation, a read-only session's first `SELECT 1` fails with the cluster's own +security error (status preserved) plus an appended message naming both routes of this +guidance. + +### What stays rejected + +The select-list must be constant scalar expressions — literals or Painless-translatable +functions of literals. Rejected with a named reason (`... requires a FROM clause`): + +- column references — `SELECT col`, including embedded ones (`SELECT UPPER(col)`) +- `SELECT *` +- aggregations (`SELECT COUNT(*)`) and window functions +- `EXCEPT(...)`, duplicate output column names, unbound `?` parameters, array literals, + negative `LIMIT`/`OFFSET` + +Rejected at the grammar level: `WHERE` / `GROUP BY` / `HAVING` / `ORDER BY` / `UNION ALL` +after a FROM-less select-list, and `DISTINCT` literals. Note that `CAST('125' AS BIGINT)` +does not parse (a pre-existing grammar gap for literal operands) — use the `'125'::BIGINT` +spelling for constant casts. + +--- + ## WHERE The `WHERE` clause supports: diff --git a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala index 7e7bc00b..e2e88cfb 100644 --- a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala +++ b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala @@ -146,6 +146,19 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { )""") } + it should "REJECT a FROM-less SELECT (issue #251 — the handshake is not a search)" in { + // Since story 20.9, `SELECT 1` PARSES (FromlessSelect). searchAs must still refuse it — + // with a clean c.abort naming the statement kind, never a macro MatchError. + assertDoesNotCompile(""" + import app.softnetwork.elastic.client.macros.TestElasticClientApi + import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats + import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product + + TestElasticClientApi.searchAs[Product]( + "SELECT 1" + )""") + } + it should "REJECT query with invalid field names" in { assertDoesNotCompile(""" import app.softnetwork.elastic.client.macros.TestElasticClientApi diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala index 986db97a..5f16caef 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala @@ -180,6 +180,18 @@ trait SQLQueryValidator { c.abort(c.enclosingPosition, "❌ Empty multi-search query") } + // Issue #251 — SELECT without FROM now PARSES (FromlessSelect, the connection + // handshake). It is not a search over documents, so searchAs/scrollAs cannot type it: + // abort cleanly naming the statement kind instead of falling through to a macro + // MatchError (a raw scalac crash). + case Right(other) => + c.abort( + c.enclosingPosition, + s"❌ Not a search statement (${other.getClass.getSimpleName}): $sqlQuery\n" + + "searchAs/scrollAs require a SELECT ... FROM ... query; " + + "run FROM-less SELECTs through GatewayApi.run instead." + ) + case Left(error) => c.abort( c.enclosingPosition, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 7b9a720d..c8800171 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -99,6 +99,13 @@ object Parser case s => MultiSearch(s) } + /** FROM-less SELECT (issue #251): the same select-list grammar, no FROM, optional LIMIT. `SELECT + * 1 LIMIT 100` must parse — it is Superset's engine probe AND what the Flight sidecar's own + * schemaProbeSql rewrites `SELECT 1` into. + */ + def fromlessSelect: PackratParser[FromlessSelect] = + select ~ limit.? ^^ { case s ~ l => FromlessSelect(s, l) } + def row: PackratParser[List[Value[_]]] = lparen ~> repsep(array_of_struct | struct | value, comma) <~ rparen @@ -1099,6 +1106,11 @@ object Parser def dqlStatement: PackratParser[DqlStatement] = { searchStatement | + // Issue #251 — FROM-less SELECT. MUST stay immediately AFTER searchStatement: `|` commits + // to the first SUCCEEDING alternative, and searchStatement FAILS (not partially succeeds) + // on FROM-less input because `single` requires `from`; placing fromlessSelect first would + // commit to the prefix parse and break every `SELECT ... FROM ...`. Do not reorder. + fromlessSelect | showTables | showTable | showCreateTable | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 674de21c..ee180102 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -402,6 +402,166 @@ package object query { ListMap(requests.flatMap(_.fieldAliases).distinct: _*) } + /** FROM-less SELECT of constant scalar expressions — the connection/health idiom of the + * JDBC/SQLAlchemy ecosystem (issue #251): Tableau re-issues `SELECT 1` on every interaction, + * Superset's connect test sends `SELECT 1` and its engine probe `SELECT 1 LIMIT 100`. + * + * EXECUTED AGAINST ELASTICSEARCH (lead direction 2026-09-02): the select-list is rewritten to a + * SingleSearch over the dedicated handshake index and rides the existing SQL to Painless + * translation as script_fields — a locally-answered handshake would report "connected" against a + * dead cluster. Deliberately NOT a SearchStatement: the PUBLIC statement must never reach + * SearchApi / the scroll machinery or the SingleSearch quota arm; only the INTERNAL rewrite + * (toSingleSearch, always LIMIT 1) executes, below those seams. + */ + case class FromlessSelect( + // NO default for `select`: Select()'s own default is `SELECT *` + // (fields = Seq(Field(Identifier("*"))), query/Select.scala) — a no-arg + // FromlessSelect() would construct exactly the statement this class rejects. + select: Select, + limit: Option[Limit] = None + ) extends DqlStatement { + + override def sql: String = s"$select${asString(limit)}" + + /** Output column names, SELECT-list order: explicit alias, else the rendered expression + * (`SELECT 1` -> "1") — the convention hosts already accept (19.4 shim, MySQL family). The + * internal `__cN` computed aliases (Select.fieldsWithComputedAliases) name the ES script + * fields only; result assembly maps them back to these names. + */ + lazy val columnNames: Seq[String] = + select.fields.map(f => f.fieldAlias.map(_.alias).getOrElse(f.identifier.identifierName)) + + /** The execution rewrite: exactly what the parser's `single` production would build for `SELECT + * FROM LIMIT 1` (Parser.single maps through .update(); programmatic + * SingleSearch(...).update() is the established #212 pattern). ALWAYS `LIMIT 1`: the + * statement's own LIMIT/OFFSET are applied on the assembled row by the executor (AD-12) and + * must never leak here — a host's LIMIT 11000 would otherwise flip `requiresScrollPaging` and + * route a handshake into scroll/PIT. + */ + def toSingleSearch(dummyIndex: String): SingleSearch = + SingleSearch( + select = select, + from = From(Seq(Table(dummyIndex))), + where = None, + limit = Some(Limit(1, None)) + ).update() + + /** ListMap rows silently collapse duplicate keys (PD-3). */ + private def checkDuplicateColumns(): Either[String, Unit] = { + val dupes = columnNames.diff(columnNames.distinct).distinct + if (dupes.nonEmpty) + Left( + s"Duplicate column name(s) ${dupes.mkString(", ")} in FROM-less SELECT: " + + "alias each expression distinctly" + ) + else Right(()) + } + + /** `Limit.validate()` is the Validation-trait default no-op and the `long` regex accepts a + * sign, so `LIMIT -5` (and a negatively-wrapping Int overflow like `LIMIT 4294967291`, + * LimitParser's `.toInt`) would silently yield zero rows through `take(negative)` — reject + * loudly instead (AD-8). Honest bound: an overflow that wraps POSITIVE is indistinguishable + * from that literal once Limit holds an Int — pre-existing LimitParser behaviour, every + * statement kind. + */ + private def checkLimit(): Either[String, Unit] = + limit match { + case Some(l) if l.limit < 0 || l.offset.exists(_.offset < 0) => + Left(s"LIMIT/OFFSET must be non-negative in FROM-less SELECT, got${l.sql}") + case _ => Right(()) + } + + /** One field's acceptance guard — the lead's rule: literals or Painless-translatable scalars + * with NO document-field reference; no window functions; no aggregations. ORDER IS + * LOAD-BEARING: hasWindow FIRST (WindowFunction extends AggregateFunction, + * function/aggregate/package.scala — aggregation-first makes the window reject dead code and + * mislabels it); then aggregation; then `*`; then the field's OWN name (a bare + * `Identifier("col")` has empty `functions` so its `dependencies` is EMPTY — + * Function.dependencies filters on the walked identifiers, not the root); then `dependencies` + * for names embedded through functions (`UPPER(col)`, `COALESCE(col,1)`); then the placeholder + * walk. + */ + private def checkField(f: Field): Either[String, Unit] = + if (f.identifier.hasWindow) + Left(s"Window function '${f.identifier.identifierName}' requires a FROM clause") + else if (f.hasAggregation) + Left(s"Aggregation '${f.identifier.identifierName}' requires a FROM clause") + else if (f.identifier.name == "*") + Left("SELECT * requires a FROM clause") + else if (f.identifier.name.nonEmpty) + Left(s"Column reference '${f.identifier.name}' requires a FROM clause") + else + f.identifier.dependencies.headOption match { + case Some(dep) => Left(s"Column reference '${dep.name}' requires a FROM clause") + case None => FromlessSelect.checkPlaceholders(f.identifier) + } + + override def validate(): Either[String, Unit] = + for { + _ <- select.except match { + case Some(_) => Left("EXCEPT(...) requires a FROM clause") + case None => Right(()) + } + _ <- checkLimit() + _ <- checkDuplicateColumns() + _ <- select.fields.foldLeft[Either[String, Unit]](Right(())) { (acc, f) => + acc.flatMap(_ => checkField(f)) + } + } yield () + } + + object FromlessSelect { + + /** Reject non-constant placeholder Values ANYWHERE in the expression tree (walk shape mirrors + * FunctionUtils.funIdentifiers / aggregateFunctions, function/package.scala): + * - ParamValue renders `params.paramValue`; a nested unbound `?` reads as painless NULL at + * ES — the silent-NULL #253 class, re-closed here for the ES path. (Top-level `SELECT ?` + * is the only host-reachable shape — JDBC substitutes `?` textually before core sees the + * SQL — but raw gateway.run callers can nest it.) + * - IdValue / IngestTimestampValue render moustache `{{...}}` templates — not query-context + * painless (programmatic-only; no select-position production). + * - Values (array literals) render multi-element painless lists, defeating the 1-element + * script_fields unwrap contract (AD-11). Every arm verified against the real classes; + * extend deliberately when a new container production appears — never by falling through. + * Arm ORDER is load-bearing: Conversion embeds its operand and overrides `args` to empty + * (before FunctionN); Identifier extends FunctionChain (before FunctionChain). + */ + private[query] def checkPlaceholders(f: function.Function): Either[String, Unit] = + f match { + case ParamValue => + Left("Unbound parameter '?' is not supported without a FROM clause") + case IdValue | IngestTimestampValue => + Left("Ingest placeholder is not supported without a FROM clause") + case vs: Values[_, _] => + Left(s"Array literal ${vs.sql} is not supported without a FROM clause") + case c: function.convert.Conversion => + // Conversion EMBEDS its operand (value: PainlessScript) and its args are empty + c.value match { + case g: function.Function => checkPlaceholders(g) + case _ => Right(()) + } + case id: Identifier => + id.functions.foldLeft[Either[String, Unit]](Right(())) { (acc, g) => + acc.flatMap(_ => checkPlaceholders(g)) + } + case fn: function.FunctionN[_, _] => + // covers BinaryFunction (args = List(left, right)), hence ArithmeticExpression + // operands, COALESCE/NULLIF/GREATEST/LEAST args, ... + fn.args.foldLeft[Either[String, Unit]](Right(())) { (acc, a) => + a match { + case g: function.Function => acc.flatMap(_ => checkPlaceholders(g)) + case _ => acc + } + } + case fwi: function.FunctionWithIdentifier => checkPlaceholders(fwi.identifier) + case fc: function.FunctionChain => + fc.functions.foldLeft[Either[String, Unit]](Right(())) { (acc, g) => + acc.flatMap(_ => checkPlaceholders(g)) + } + case _ => Right(()) + } + } + sealed trait DmlStatement extends Statement case class OnConflict(target: Option[Seq[String]], doUpdate: Boolean) extends Token { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/FromlessSelectParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/FromlessSelectParserSpec.scala new file mode 100644 index 00000000..0f00e15e --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/FromlessSelectParserSpec.scala @@ -0,0 +1,115 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.{FromlessSelect, SingleSearch} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 20.9 / issue #251 — FROM-less SELECT (the connection handshake). + * + * Parse-first discipline: every assertion below is pinned against real parser output, never a + * guessed AST shape. Reason substrings are pinned ONLY where this story authors the reason + * (validate() messages); grammar-internal wording is never pinned (20.4 F-4). + */ +class FromlessSelectParserSpec extends AnyFlatSpec with Matchers { + + private def parseFromless(sql: String): FromlessSelect = + Parser(sql) match { + case Right(f: FromlessSelect) => f + case other => fail(s"expected FromlessSelect for [$sql], got $other") + } + + // ── host-idiom minimum (AC 1) ────────────────────────────────────────────── + "FROM-less SELECT" should "parse the Tableau/Superset handshake idioms" in { + val shapes = + Seq( + "SELECT 1", + "SELECT 1;", + "select 1", + "SELECT 1 AS x", + "SELECT 1 x", + "SELECT 1 LIMIT 100" + ) + shapes.foreach { s => parseFromless(s) } + parseFromless("SELECT 1 AS x").columnNames shouldBe Seq("x") + parseFromless("SELECT 1 x").columnNames shouldBe Seq("x") // bare alias == AS alias + parseFromless("SELECT 1").columnNames shouldBe Seq("1") + } + + it should "not capture LIMIT as an alias" in { + val f = parseFromless("SELECT 1 LIMIT 100") + f.columnNames shouldBe Seq("1") + f.limit.map(_.limit) shouldBe Some(100) + } + + it should "leave FROM-bearing SELECTs on the search path" in { + Parser("SELECT 1 FROM dual") match { + case Right(_: SingleSearch) => succeed + case other => fail(s"regression: $other") + } + } + + // ── fixed point (AC 10, #218 discipline) ─────────────────────────────────── + it should "round-trip every accepted form through its own .sql render" in { + val accepted = Seq( + "SELECT 1", + "SELECT 1 AS x", + "SELECT 1 x", + "SELECT 1 LIMIT 100", + "SELECT -1", + "SELECT 1.5", + "SELECT true", + "SELECT NULL AS n", + "SELECT 1+1", + "SELECT (1+2)*3 AS nine", + "SELECT 3/2.0 AS r", + "SELECT 1/0 AS boom", // parses — no local evaluation; fails at EXECUTION on ES + "SELECT CURRENT_TIMESTAMP AS ts", + "SELECT CURRENT_DATE AS d", + "SELECT '125'::BIGINT AS c", // CAST('125' AS BIGINT) does NOT parse — OQ-5 + "SELECT COALESCE(NULL, 1) AS c", + "SELECT UPPER('ok') AS u", + "SELECT LENGTH('abc') AS l", + "SELECT ABS(-5) AS a", + "SELECT PI", + "SELECT RANDOM AS r", + "SELECT 3000000000 AS big", + "SELECT 1 AS a, 'x' AS b, true AS c", + "SELECT 1 AS ok, UPPER('x') AS u, 1+1 AS two", // the documentation example, parse-probed + // OQ-2 rows, dev-verified 2026-09-02: newly reachable under PD-5, parse + guards hold + "SELECT CURRENT_DATE - INTERVAL 1 DAY AS d", + "SELECT CASE WHEN 1 = 1 THEN 'a' ELSE 'b' END AS c" + ) + accepted.foreach { s => + val first = parseFromless(s) + withClue(s"render [${first.sql}] of [$s]: ") { + Parser(first.sql).toOption should contain(first) + } + } + } + + // ── rejects (AC 3) — reason pins only where WE author the reason ─────────── + it should "reject column references, star, aggregates, placeholders with named reasons" in { + def leftMsg(sql: String): String = + Parser(sql).swap.getOrElse(fail(s"[$sql] unexpectedly parsed")).msg + leftMsg("SELECT col") should include("Column reference 'col' requires a FROM clause") + leftMsg("SELECT *") should include("SELECT * requires a FROM clause") + leftMsg("SELECT COUNT(*)") should include("requires a FROM clause") + leftMsg("SELECT 1, 1") should include("Duplicate column name") + leftMsg("SELECT a + 1") should include("Column reference 'a'") + leftMsg("SELECT ?") should include("Unbound parameter") + leftMsg("SELECT COALESCE(?, 1)") should include("Unbound parameter") // nested — walk depth + leftMsg("SELECT 1 LIMIT -5") should include("must be non-negative") + } + + it should "keep grammar-level rejects rejected (message unpinned — F-4)" in { + Seq( + "SELECT 1 WHERE 1=1", + "SELECT 1 ORDER BY 1", + "SELECT 1 GROUP BY 1", + "SELECT 1 HAVING 1=1", + "SELECT 1 UNION ALL SELECT 2", + "SELECT DISTINCT 1", + "SELECT 1 AS `x`" // backtick aliases stay #252's acceptance row + ).foreach { s => withClue(s"[$s]: ")(Parser(s).isLeft shouldBe true) } + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/FromlessSelectValidateSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/FromlessSelectValidateSpec.scala new file mode 100644 index 00000000..8c79588d --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/FromlessSelectValidateSpec.scala @@ -0,0 +1,105 @@ +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.parser.Parser +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 20.9 / issue #251 — FromlessSelect guard arms + the toSingleSearch rewrite pins. + * + * Replaces the struck ConstantFoldingSpec (Wave A″): semantics now live in ES, so this spec pins + * only what the sql module owns — the validate() guard matrix (parse-first: every row parsed with + * the real Parser, never a hand-built AST unless the point IS the programmatic path) and the shape + * of the internal SingleSearch rewrite. + */ +class FromlessSelectValidateSpec extends AnyFlatSpec with Matchers { + + private val idx = "softclient4es_handshake" + + private def parseFromless(sql: String): FromlessSelect = + Parser(sql) match { + case Right(f: FromlessSelect) => f + case other => fail(s"expected FromlessSelect for [$sql], got $other") + } + + private def leftMsg(sql: String): String = + Parser(sql).swap.getOrElse(fail(s"[$sql] unexpectedly parsed")).msg + + // ── the rewrite (AD-3′) ──────────────────────────────────────────────────── + "FromlessSelect.toSingleSearch" should "build the parser-equivalent SingleSearch with LIMIT 1" in { + val single = parseFromless("SELECT 1").toSingleSearch(idx) + single.sql shouldBe s"SELECT 1 FROM $idx LIMIT 1" + single.limit shouldBe Some(Limit(1, None)) + // every constant select item IS a script field — the FROM-ful script_fields pipeline + single.scriptFields.size shouldBe 1 + single.returnsRows shouldBe true + // the ES response keys are the computed aliases; assembly maps them back to PD-2 names + single.select.fieldsWithComputedAliases.head.fieldAlias.map(_.alias) shouldBe Some("__c1") + } + + it should "round-trip its own render through the real parser (generated-SQL fixed point)" in { + Seq( + "SELECT 1", + "SELECT 1 AS x", + "SELECT UPPER('ok') AS u", + "SELECT CURRENT_TIMESTAMP AS ts, CURRENT_TIMESTAMP AS ts2" + ).foreach { s => + val rewritten = parseFromless(s).toSingleSearch(idx) + withClue(s"rewrite [${rewritten.sql}] of [$s]: ") { + Parser(rewritten.sql).toOption should contain(rewritten) + } + } + } + + it should "never leak the statement's LIMIT into the rewrite" in { + // AD-12: a host's LIMIT must never reach the rewrite — LIMIT 11000 would flip + // requiresScrollPaging and route a handshake into scroll/PIT. + parseFromless("SELECT 1 LIMIT 100").toSingleSearch(idx).limit shouldBe Some(Limit(1, None)) + parseFromless("SELECT 1 LIMIT 11000 OFFSET 5") + .toSingleSearch(idx) + .limit shouldBe Some(Limit(1, None)) + } + + // ── guard arms (parse-first, PD-5 reject matrix) ─────────────────────────── + "FromlessSelect.validate" should "reject column references by own name and via dependencies" in { + leftMsg("SELECT col") should include("Column reference 'col' requires a FROM clause") + leftMsg("SELECT UPPER(col)") should include("Column reference") + leftMsg("SELECT COALESCE(col, 1)") should include("Column reference") + leftMsg("SELECT a + 1") should include("Column reference 'a'") + // EValue is grammar-unreachable (value = literal|pi|random|...) — E is a column reference + leftMsg("SELECT E") should include("Column reference 'E'") + } + + it should "reject star, aggregates and windows — window checked BEFORE aggregation" in { + leftMsg("SELECT *") should include("SELECT * requires a FROM clause") + leftMsg("SELECT COUNT(*)") should include("requires a FROM clause") + // WindowFunction extends AggregateFunction: aggregation-first would make this arm dead code + val windowMsg = leftMsg("SELECT ROW_NUMBER() OVER (ORDER BY salary DESC) AS rn") + windowMsg should include("Window function") + (windowMsg should not).include("Aggregation") + } + + it should "reject placeholders anywhere in the tree, array literals, EXCEPT and LIMIT bounds" in { + leftMsg("SELECT ?") should include("Unbound parameter") + leftMsg("SELECT COALESCE(?, 1)") should include("Unbound parameter") // nested — tree walk + leftMsg("SELECT 1, 1") should include("Duplicate column name") + leftMsg("SELECT 1 LIMIT -5") should include("must be non-negative") + // LimitParser does .toInt — 4294967291 wraps to -5; without the guard the result would + // silently empty through take(negative) (#253 family). NOTE the guard's honest bound: + // an overflow that wraps POSITIVE (e.g. 9999999999 -> 1410065407) is indistinguishable + // from that literal by the time Limit holds an Int — pre-existing LimitParser behaviour, + // every statement kind, out of 20.9 scope. + leftMsg("SELECT 1 LIMIT 4294967291") should include("must be non-negative") + leftMsg("SELECT 1 EXCEPT(a)") should include("EXCEPT(...) requires a FROM clause") + // array literal — multi-element painless lists defeat the 1-element unwrap contract (AD-11); + // parse-first: whichever seam rejects it (grammar or validate), it must be a Left + Parser("SELECT ['a', 'b']").isLeft shouldBe true + } + + it should "guard the programmatic path too — built WITHOUT the parser" in { + // run(statement) never calls validate(): the guard set must be self-contained. + FromlessSelect(Select(Seq(Field(Identifier("*"))))).validate().isLeft shouldBe true + FromlessSelect(Select(Seq(Field(Identifier("col"))))).validate() shouldBe + Left("Column reference 'col' requires a FROM clause") + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index 370231ad..880f2adc 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client -import app.softnetwork.elastic.client.result.DmlResult +import app.softnetwork.elastic.client.result.{DmlResult, ElasticSuccess} import app.softnetwork.elastic.scalatest.ElasticTestKit import app.softnetwork.elastic.sql.{DoubleValue, IdValue} import app.softnetwork.elastic.sql.`type`.SQLTypes @@ -1471,6 +1471,100 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { assertDdl(System.nanoTime(), client.run(sql).futureValue) } + // =========================================================================== + // 5b. FROM-less SELECT — the connection handshake (story 20.9 / issue #251) + // =========================================================================== + + behavior of "FROM-less SELECT handshake" + + it should "answer the FROM-less connection handshake against the live cluster" in { + // issue #251 — the Tableau/Superset connect idiom, executed as Painless AGAINST ES + var rows = assertQueryRows(System.nanoTime(), client.run("SELECT 1").futureValue) + rows shouldBe Seq(Map("1" -> 1)) // Integer via Jackson — NOT List(1): AC 7's unwrap + // PD-4: the value round-trips ES JSON as Jackson's smallest type — INTEGER, not BIGINT + // (Scala's cooperative equality makes `shouldBe 1` pass for a Long too — pin the class) + rows.head("1").isInstanceOf[Int] shouldBe true + rows = assertQueryRows(System.nanoTime(), client.run("SELECT 1 LIMIT 100").futureValue) + rows.size shouldBe 1 + rows = assertQueryRows( + System.nanoTime(), + client.run("SELECT 1 AS ok, UPPER('x') AS u, 1+1 AS two").futureValue + ) + rows.head("ok") shouldBe 1 + rows.head("u") shouldBe "X" + rows.head("two") shouldBe 2 + // constant cast — '125'::BIGINT arrives as the VALUE's own type (Jackson smallest — PD-4) + rows = assertQueryRows( + System.nanoTime(), + client.run("SELECT '125'::BIGINT AS c").futureValue + ) + rows.head("c") shouldBe 125 + // OQ-2 rows (PD-5, dev-verified): interval arithmetic and CASE over constants ride the + // same FROM-ful painless pipeline — value truth asserted on the live cluster + rows = assertQueryRows( + System.nanoTime(), + client.run("SELECT CASE WHEN 1 = 1 THEN 'a' ELSE 'b' END AS c").futureValue + ) + rows.head("c") shouldBe "a" + rows = assertQueryRows( + System.nanoTime(), + client.run("SELECT CURRENT_DATE - INTERVAL 1 DAY AS d").futureValue + ) + Option(rows.head("d")) should not be None + } + + it should "return NULL, fresh RANDOM values and honour LIMIT 0 on the handshake path" in { + // NULL literal — the empty/absent-fields assembly on real ES (AC 7) + var rows = assertQueryRows(System.nanoTime(), client.run("SELECT NULL AS n").futureValue) + rows.size shouldBe 1 + Option(rows.head("n")) shouldBe None + // RANDOM is painless Math.random() — fresh per execution (AD-7′) + val r1 = + assertQueryRows(System.nanoTime(), client.run("SELECT RANDOM AS r").futureValue).head("r") + val r2 = + assertQueryRows(System.nanoTime(), client.run("SELECT RANDOM AS r").futureValue).head("r") + r1 should not be r2 + // LIMIT 0 returns zero rows but the round-trip still executed (AD-8) + rows = assertQueryRows(System.nanoTime(), client.run("SELECT 1 LIMIT 0").futureValue) + rows shouldBe Seq.empty + } + + it should "latch one clock per FROM-less statement" in { + // AD-6′ — one __now__ per search request, byte-equal items (the C7 guarantee, re-derived) + val rows = assertQueryRows( + System.nanoTime(), + client.run("SELECT CURRENT_TIMESTAMP AS a, CURRENT_TIMESTAMP AS b").futureValue + ) + rows.head("a") shouldBe rows.head("b") + } + + it should "fail loudly at execution on a broken constant script" in { + // SELECT 1/0 PARSES (no local evaluation any more) and fails on ES with a script error — + // loud, inherited FROM-ful semantics. Message deliberately unpinned (grammar/ES-internal). + val res = client.run("SELECT 1/0").futureValue + res.isFailure shouldBe true + } + + it should "keep the handshake index invisible to SHOW TABLES while it exists" in { + // the handshake tests above ran => the index exists... + client.indexExists(GatewayApi.HandshakeIndex, pattern = false) shouldBe ElasticSuccess(true) + // ...but no SHOW TABLES pattern ever lists it (AC 6 — the ONE seam covering jdbc getTables + // and Flight GET_TABLES; the ':74' dot-pattern assertion above is untouched: non-dot name). + val all = assertQueryRows(System.nanoTime(), client.run("SHOW TABLES").futureValue) + all.map(_("name")) should not contain GatewayApi.HandshakeIndex + val like = assertQueryRows( + System.nanoTime(), + client.run("SHOW TABLES LIKE 'softclient4es%'").futureValue + ) + like shouldBe Seq.empty + // DESCRIBE TABLE deliberately still works on it (debuggability — AD-10) + val describe = assertQueryRows( + System.nanoTime(), + client.run(s"DESCRIBE TABLE ${GatewayApi.HandshakeIndex}").futureValue + ) + describe.exists(_("Field") == "dummy") shouldBe true + } + // =========================================================================== // 6. ERRORS — parsing errors, unsupported SQL // =========================================================================== diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala index ea5d8f22..cee6a281 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala @@ -1261,6 +1261,19 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { assertDdl(System.nanoTime(), executeSync("DROP PIPELINE IF EXISTS user_pipeline")) } + // ========================================================================= + // 6b. FROM-less SELECT — the connection handshake (story 20.9 / issue #251) + // ========================================================================= + + behavior of "REPL - FROM-less SELECT handshake" + + it should "answer SELECT 1 through the REPL gateway path" in { + // issue #251 — the connect idiom the 20.8 live-acceptance venue exercises: one row, + // column named "1", INTEGER value, executed AGAINST the live cluster. + val rows = assertQueryRows(System.nanoTime(), executeSync("SELECT 1")) + rows shouldBe Seq(Map("1" -> 1)) + } + // ========================================================================= // 7. Error handling // =========================================================================