From 428e1ace47b6d142b4d4714dfed8772389a03aee Mon Sep 17 00:00:00 2001 From: Ben Dodson Date: Wed, 26 Aug 2026 02:01:19 -0700 Subject: [PATCH] feat(clientsql): add portable runtime and debugger --- BUILD.bazel | 10 + MODULE.bazel | 16 + bin/BUILD.bazel | 17 +- bzl/BUILD.bazel | 6 + bzl/dependencies.bzl | 16 + bzl/valdi/BUILD.bazel | 17 +- bzl/valdi/valdi_compiled.bzl | 4 + bzl/valdi/valdi_run_compiler.bzl | 6 + bzl/valdi/valdi_toolchain.bzl | 10 +- compiler/clientsql/BUILD.bazel | 19 + compiler/clientsql/README.md | 58 +- compiler/clientsql/sqlite_316_validator.cpp | 403 ++++ compiler/clientsql/src/clientsql/cli.py | 97 +- compiler/clientsql/src/clientsql/sql.py | 278 ++- .../clientsql/src/clientsql/typescript.py | 183 +- compiler/clientsql/src/clientsql/validator.py | 178 ++ compiler/clientsql/test_clientsql.py | 640 ++++++- compiler/compiler/BUILD.bazel | 10 + .../Sources/Config/ValdiProjectConfig.swift | 13 + .../Processors/ClientSqlProcessor.swift | 47 +- .../Sources/ValdiCompilerArguments.swift | 3 + .../ClientSqlProcessorTests.swift | 27 + fossa-deps.yml | 6 + .../src/valdi/client_sql/BUILD.bazel | 86 + .../src/valdi/client_sql/README.md | 33 + .../src/valdi/client_sql/module.yaml | 8 + .../native/ClientSQLNativeModuleFactory.cpp | 1670 +++++++++++++++++ .../native/ClientSQLNativeModuleFactory.hpp | 24 + .../ClientSQLNativeModuleFactory_tests.cpp | 1274 +++++++++++++ .../src/valdi/client_sql/src/ClientSQL.ts | 5 + .../valdi/client_sql/src/ClientSQLDebug.ts | 928 +++++++++ .../valdi/client_sql/src/ClientSQLNative.d.ts | 112 ++ .../client_sql/test/ClientSQLDebug.spec.ts | 219 +++ .../src/valdi/client_sql/tsconfig.json | 8 + .../valdi/client_sql/web/ClientSQLNative.ts | 77 + .../src/valdi/client_sql/web/tsconfig.json | 16 + third-party/sqlite/BUILD.bazel | 11 + third-party/sqlite/LICENSE.md | 85 + third-party/sqlite/README.md | 44 + third-party/sqlite/sqlite.BUILD | 24 + third-party/sqlite/sqlite_316.BUILD | 27 + valdi/BUILD.bazel | 41 +- .../integration/ClientSQLRuntime_tests.cpp | 126 ++ valdi/test/integration/RuntimeTestsUtils.cpp | 16 +- valdi/test/integration/RuntimeTestsUtils.hpp | 12 +- .../modules/client_sql_smoke/BUILD.bazel | 33 + .../modules/client_sql_smoke/module.yaml | 9 + .../client_sql_smoke/sql/TestDb/User.sq | 29 + .../client_sql_smoke/sql/migration/2.sqm | 1 + .../client_sql_smoke/src/ClientSQLSmoke.ts | 116 ++ .../modules/client_sql_smoke/tsconfig.json | 8 + 51 files changed, 6921 insertions(+), 185 deletions(-) create mode 100644 compiler/clientsql/sqlite_316_validator.cpp create mode 100644 compiler/clientsql/src/clientsql/validator.py create mode 100644 compiler/compiler/Compiler/Tests/CompilerTests/ClientSqlProcessorTests.swift create mode 100644 src/valdi_modules/src/valdi/client_sql/BUILD.bazel create mode 100644 src/valdi_modules/src/valdi/client_sql/README.md create mode 100644 src/valdi_modules/src/valdi/client_sql/module.yaml create mode 100644 src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.cpp create mode 100644 src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.hpp create mode 100644 src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory_tests.cpp create mode 100644 src/valdi_modules/src/valdi/client_sql/src/ClientSQL.ts create mode 100644 src/valdi_modules/src/valdi/client_sql/src/ClientSQLDebug.ts create mode 100644 src/valdi_modules/src/valdi/client_sql/src/ClientSQLNative.d.ts create mode 100644 src/valdi_modules/src/valdi/client_sql/test/ClientSQLDebug.spec.ts create mode 100644 src/valdi_modules/src/valdi/client_sql/tsconfig.json create mode 100644 src/valdi_modules/src/valdi/client_sql/web/ClientSQLNative.ts create mode 100644 src/valdi_modules/src/valdi/client_sql/web/tsconfig.json create mode 100644 third-party/sqlite/BUILD.bazel create mode 100644 third-party/sqlite/LICENSE.md create mode 100644 third-party/sqlite/README.md create mode 100644 third-party/sqlite/sqlite.BUILD create mode 100644 third-party/sqlite/sqlite_316.BUILD create mode 100644 valdi/test/integration/ClientSQLRuntime_tests.cpp create mode 100644 valdi/testdata/resources/modules/client_sql_smoke/BUILD.bazel create mode 100644 valdi/testdata/resources/modules/client_sql_smoke/module.yaml create mode 100644 valdi/testdata/resources/modules/client_sql_smoke/sql/TestDb/User.sq create mode 100644 valdi/testdata/resources/modules/client_sql_smoke/sql/migration/2.sqm create mode 100644 valdi/testdata/resources/modules/client_sql_smoke/src/ClientSQLSmoke.ts create mode 100644 valdi/testdata/resources/modules/client_sql_smoke/tsconfig.json diff --git a/BUILD.bazel b/BUILD.bazel index cbc8ae60e..1d6672154 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,3 +12,13 @@ npm_link_package( src = "@valdi//src/valdi_modules/src/valdi/valdi_core:valdi_core_dts", visibility = ["//visibility:public"], ) + +filegroup( + name = "clientsql_generator_test_data", + srcs = [ + "MODULE.bazel", + "fossa-deps.yml", + ], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/MODULE.bazel b/MODULE.bazel index 2af2510bc..5cb21514d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -193,6 +193,22 @@ http_archive( url = "https://github.com/fmtlib/fmt/releases/download/7.1.3/fmt-7.1.3.zip", ) +http_archive( + name = "sqlite", + build_file = "@valdi//third-party/sqlite:sqlite.BUILD", + sha256 = "0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c", + strip_prefix = "sqlite-autoconf-3530400", + url = "https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz", +) + +http_archive( + name = "sqlite_316", + build_file = "@valdi//third-party/sqlite:sqlite_316.BUILD", + sha256 = "3b5dfb65807e2b17e6463357df848e322badba01dc9a4a1de8fdbb72d448e3b0", + strip_prefix = "sqlite-amalgamation-3160000", + url = "https://www.sqlite.org/2017/sqlite-amalgamation-3160000.zip", +) + bazel_dep(name = "android_macros") local_path_override( module_name = "android_macros", diff --git a/bin/BUILD.bazel b/bin/BUILD.bazel index c13be7d46..91747c9ca 100644 --- a/bin/BUILD.bazel +++ b/bin/BUILD.bazel @@ -6,9 +6,15 @@ load( "valdi_compiler_companion_files", ) -filegroup( +alias( name = "sqldelight_compiler", - srcs = [], + actual = "@valdi//compiler/clientsql:clientsql", + visibility = ["//visibility:public"], +) + +alias( + name = "clientsql_sqlite_validator", + actual = "@valdi//compiler/clientsql:sqlite_316_validator", visibility = ["//visibility:public"], ) @@ -52,3 +58,10 @@ js_binary( visibility = ["//visibility:public"], # log_level = "debug", # increase verbosity of logs if you need to debug anything ) + +filegroup( + name = "clientsql_generator_test_data", + srcs = ["BUILD.bazel"], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/bzl/BUILD.bazel b/bzl/BUILD.bazel index e69de29bb..2d927e575 100644 --- a/bzl/BUILD.bazel +++ b/bzl/BUILD.bazel @@ -0,0 +1,6 @@ +filegroup( + name = "clientsql_generator_test_data", + srcs = ["dependencies.bzl"], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/bzl/dependencies.bzl b/bzl/dependencies.bzl index a6d75bbe2..15d1852aa 100644 --- a/bzl/dependencies.bzl +++ b/bzl/dependencies.bzl @@ -222,6 +222,22 @@ def setup_dependencies(workspace_root = None): url = "https://github.com/fmtlib/fmt/releases/download/7.1.3/fmt-7.1.3.zip", ) + http_archive( + name = "sqlite", + build_file = "@valdi//third-party/sqlite:sqlite.BUILD", + sha256 = "0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c", + strip_prefix = "sqlite-autoconf-3530400", + url = "https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz", + ) + + http_archive( + name = "sqlite_316", + build_file = "@valdi//third-party/sqlite:sqlite_316.BUILD", + sha256 = "3b5dfb65807e2b17e6463357df848e322badba01dc9a4a1de8fdbb72d448e3b0", + strip_prefix = "sqlite-amalgamation-3160000", + url = "https://www.sqlite.org/2017/sqlite-amalgamation-3160000.zip", + ) + local_or_nested_repository( workspace_root = workspace_root, name = "android_macros", diff --git a/bzl/valdi/BUILD.bazel b/bzl/valdi/BUILD.bazel index 9b4475e3c..0af3a9e45 100644 --- a/bzl/valdi/BUILD.bazel +++ b/bzl/valdi/BUILD.bazel @@ -434,7 +434,11 @@ valdi_toolchain( compiler = ":compiler_source", compiler_companion = ":compiler_companion_source", compiler_toolbox = "@valdi_toolchain//:valdi_compiler_toolbox", - sqldelight_compiler = "@valdi_toolchain//:sqldelight_compiler", + # ClientSQL is built from the reviewed generator sources in this repository. + # Keeping it in the exec configuration gives local and remote actions the + # same hermetic executable and runfiles rather than an unpublished prebuilt. + sqldelight_compiler = "//compiler/clientsql:clientsql", + clientsql_sqlite_validator = "//compiler/clientsql:sqlite_316_validator", ) # Cross-compilation setups whose first execution platform is a remote linux worker set this @@ -469,3 +473,14 @@ toolchain( toolchain = ":valdi", toolchain_type = ":toolchain_type", ) + +filegroup( + name = "clientsql_generator_test_data", + srcs = [ + "BUILD.bazel", + "valdi_run_compiler.bzl", + "valdi_toolchain.bzl", + ], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/bzl/valdi/valdi_compiled.bzl b/bzl/valdi/valdi_compiled.bzl index d4c72c38c..a54b94a9b 100644 --- a/bzl/valdi/valdi_compiled.bzl +++ b/bzl/valdi/valdi_compiled.bzl @@ -2200,6 +2200,7 @@ def _prepare_hotreload_arguments(module_names, config_yaml_file, explicit_input_ compiler_toolbox = toolchain.compiler_toolbox.files.to_list()[0] minify_config = toolchain.minify_config.files.to_list()[0] client_sql = toolchain.sqldelight_compiler.files.to_list() + client_sql_validator = toolchain.clientsql_sqlite_validator.files.to_list() args = [] @@ -2220,6 +2221,9 @@ def _prepare_hotreload_arguments(module_names, config_yaml_file, explicit_input_ if client_sql: args.append("--direct-client-sql-path") args.append(client_sql[0].path) + if client_sql_validator: + args.append("--direct-client-sql-validator-path") + args.append(client_sql_validator[0].path) args.append("--module") args.append(_VALDI_BASE_MODULE_NAME) diff --git a/bzl/valdi/valdi_run_compiler.bzl b/bzl/valdi/valdi_run_compiler.bzl index f3258fdb6..b0e65e9a8 100644 --- a/bzl/valdi/valdi_run_compiler.bzl +++ b/bzl/valdi/valdi_run_compiler.bzl @@ -48,6 +48,7 @@ def resolve_compiler_executable(ctx, toolchain, include_tools): * companion tool (see //src/valdi_internal/compiler/companion:bin_wrapper) * minify config file * sqldelight compiler binary + * ClientSQL SQLite 3.16 validator binary Args: ctx: The context of the current rule invocation @@ -69,8 +70,10 @@ def resolve_compiler_executable(ctx, toolchain, include_tools): inputs_depsets.append(toolchain.compiler_toolbox.files) inputs_depsets.append(toolchain.minify_config.files) inputs_depsets.append(toolchain.sqldelight_compiler.files) + inputs_depsets.append(toolchain.clientsql_sqlite_validator.files) tools.append(toolchain.sqldelight_compiler[DefaultInfo].files_to_run) + tools.append(toolchain.clientsql_sqlite_validator[DefaultInfo].files_to_run) return (toolchain.compiler.files.to_list()[0], depset(transitive = inputs_depsets), tools) @@ -86,6 +89,7 @@ def run_valdi_compiler(ctx, args, outputs, inputs, mnemonic, progress_message, u compiler_toolbox = toolchain.compiler_toolbox.files.to_list()[0] minify_config = toolchain.minify_config.files.to_list()[0] client_sql = toolchain.sqldelight_compiler.files.to_list() + client_sql_validator = toolchain.clientsql_sqlite_validator.files.to_list() args.add("--bazel") args.add("--direct-companion-path", companion_bin_wrapper) @@ -94,6 +98,8 @@ def run_valdi_compiler(ctx, args, outputs, inputs, mnemonic, progress_message, u if client_sql: args.add("--direct-client-sql-path", client_sql[0]) + if client_sql_validator: + args.add("--direct-client-sql-validator-path", client_sql_validator[0]) env = { # required for the companion app execution under Bazel diff --git a/bzl/valdi/valdi_toolchain.bzl b/bzl/valdi/valdi_toolchain.bzl index 68b9f28e2..2979ce5b4 100644 --- a/bzl/valdi/valdi_toolchain.bzl +++ b/bzl/valdi/valdi_toolchain.bzl @@ -8,6 +8,7 @@ ValdiCompilerInfo = provider( "companion", "minify_config", "sqldelight_compiler", + "clientsql_sqlite_validator", ], ) @@ -19,6 +20,7 @@ def _valdi_toolchain_impl(ctx): companion = ctx.attr.compiler_companion, minify_config = ctx.attr.minify_config, sqldelight_compiler = ctx.attr.sqldelight_compiler, + clientsql_sqlite_validator = ctx.attr.clientsql_sqlite_validator, ), ) return [info] @@ -49,8 +51,14 @@ valdi_toolchain = rule( default = "//modules:minify_config", ), "sqldelight_compiler": attr.label( + executable = True, + cfg = "exec", + doc = "The ClientSQL generator executable to use.", + ), + "clientsql_sqlite_validator": attr.label( + executable = True, cfg = "exec", - doc = "The sqldelight compiler to use.", + doc = "The hermetic SQLite 3.16 validator used by the ClientSQL generator.", ), }, ) diff --git a/compiler/clientsql/BUILD.bazel b/compiler/clientsql/BUILD.bazel index 3d694ea13..881c52b6f 100644 --- a/compiler/clientsql/BUILD.bazel +++ b/compiler/clientsql/BUILD.bazel @@ -1,6 +1,13 @@ load("@aspect_rules_js//js:defs.bzl", "js_binary") load("@rules_python//python:defs.bzl", "py_test") +cc_binary( + name = "sqlite_316_validator", + srcs = ["sqlite_316_validator.cpp"], + visibility = ["//visibility:public"], + deps = ["@sqlite_316//:sqlite"], +) + py_library( name = "clientsql_lib", srcs = glob(["src/clientsql/*.py"]), @@ -37,12 +44,24 @@ py_test( ":clientsql", ":clientsql_test_javascript_runner", ":package_clientsql", + ":sqlite_316_validator", + "sqlite_316_validator.cpp", "@npm_typescript//:tsc", + "//:clientsql_generator_test_data", + "//bin:clientsql_generator_test_data", + "//bzl:clientsql_generator_test_data", + "//bzl/valdi:clientsql_generator_test_data", + "//compiler/compiler:clientsql_generator_test_data", + "//src/valdi_modules/src/valdi/client_sql:clientsql_generator_test_data", + "//third-party/sqlite:clientsql_generator_test_data", + "//valdi:clientsql_generator_test_data", + "//valdi/testdata/resources/modules/client_sql_smoke:clientsql_generator_test_data", ], env = { "BAZEL_BINDIR": ".", "CLIENTSQL_TEST_GENERATOR": "$(rootpath :clientsql)", "CLIENTSQL_TEST_JAVASCRIPT_RUNNER": "$(rootpath :clientsql_test_javascript_runner)", + "CLIENTSQL_TEST_SQLITE_VALIDATOR": "$(rootpath :sqlite_316_validator)", "CLIENTSQL_TEST_TYPESCRIPT_COMPILER": "$(rootpath @npm_typescript//:tsc)", }, size = "medium", diff --git a/compiler/clientsql/README.md b/compiler/clientsql/README.md index 65c28758d..61f27a31a 100644 --- a/compiler/clientsql/README.md +++ b/compiler/clientsql/README.md @@ -10,11 +10,12 @@ The source is divided by responsibility: - `sql.py` parses and validates SQL, migrations, parameters, and result shapes. - `typescript.py` emits generated TypeScript bindings and database classes. -The public Valdi toolchain continues to supply its ClientSQL executable through -the existing `sqldelight_compiler` target. This source package intentionally -does not replace that toolchain binary or check in a generated executable. Use -the Bazel `//compiler/clientsql:clientsql` target for source builds, or create a -deterministic standalone zipapp at an explicit local path: +The public Valdi toolchain uses the Bazel `//compiler/clientsql:clientsql` +`py_binary` as its canonical `sqldelight_compiler` executable. The reviewed +sources above are therefore the actual input to local and remote ClientSQL +generation; no Git LFS generator binary or unpublished toolchain release is +required. A deterministic standalone zipapp can still be created at an +explicit local path: ```bash python3 compiler/clientsql/package_clientsql.py --output /tmp/clientsql @@ -25,3 +26,50 @@ An explicitly supplied executable can be checked against the canonical source: ```bash python3 compiler/clientsql/package_clientsql.py --output /tmp/clientsql --check ``` + +Both the source entry point and zipapp require the same explicit floor-validator +path: + +```bash +python3 compiler/clientsql/src/clientsql_main.py \ + --sqlite-validator /path/to/sqlite_316_validator -version +/tmp/clientsql --sqlite-validator /path/to/sqlite_316_validator -version +``` + +The Bazel Valdi toolchain builds `//compiler/clientsql:sqlite_316_validator` in +the execution configuration and passes its path through the compiler. Direct +invocations fail closed if the path is absent, cannot execute, or reports a +different SQLite version/source identity. + +`clientsql -version` includes a SHA-256 digest over the canonical package +sources plus the exact validator executable's SHA-256, SQLite version, official +source ID, `sqlite3.c` SHA-1, and protocol version. `ClientSqlProcessor` stores +that full identity in its disk-cache metadata, so either source or validator +changes invalidate generated outputs even when the human release number remains +`0.2.0`. +Generator tests execute both the source entry point and packaged zipapp against +the same fixture and require identical version identities and generated bytes. +For a source-tree test run, first build or otherwise supply the verified target +and set `CLIENTSQL_TEST_SQLITE_VALIDATOR` to that executable; the tests do not +fall back to the host Python SQLite library. + +SQL generation targets the SQLite 3.16.0 runtime floor supported by Apple system +SQLite. Schema statements are prepared and executed, generated queries are +prepared as `EXPLAIN` with their exact parameter counts, and every migration +statement is prepared by a validator actually linked to the verified 3.16.0 +amalgamation. There is no host-Python SQLite dependency and no finite syntax +blacklist. Non-Apple runtime builds continue to use the separate pinned SQLite +3.53.4 release documented in `third-party/sqlite/README.md`. + +Generated database classes keep the caller-facing database name for display, +but derive a canonical lowercase ASCII storage identity from the Valdi module +digest plus a collision-free UTF-16 encoding of that name. The compiler passes +the canonical Valdi bundle name separately from the database package/class, so +identically named databases in different bundles receive different module +digests. This prevents two +modules from sharing a file accidentally and prevents filesystem case, +normalization, separator, or trailing-character aliases. Names are limited to +48 UTF-16 code units and may not contain control characters. Schema versions +are positive 32-bit integers; migration filenames must be unique, contiguous +32-bit versions beginning at 2. Concurrent handles for one storage identity +must present the same length-delimited schema fingerprint. diff --git a/compiler/clientsql/sqlite_316_validator.cpp b/compiler/clientsql/sqlite_316_validator.cpp new file mode 100644 index 000000000..f4979a0bb --- /dev/null +++ b/compiler/clientsql/sqlite_316_validator.cpp @@ -0,0 +1,403 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr char kProtocolHeader[] = "VALDI_CLIENTSQL_SQL_VALIDATOR_V1\n"; +constexpr char kExpectedSQLiteVersion[] = "3.16.0"; +constexpr int kExpectedSQLiteVersionNumber = 3016000; +constexpr char kExpectedSQLiteSourceId[] = + "2017-01-02 11:57:58 04ac0b75b1716541b2b97704f4809cb7ef19cccf"; +constexpr char kExpectedSQLite3CSha1[] = "e2920fb885569d14197c9b7958e6f1db573ee669"; +constexpr std::uint32_t kMaximumStatements = 100000; +constexpr std::uint32_t kMaximumStatementBytes = 16 * 1024 * 1024; +constexpr std::size_t kMaximumRequestBytes = 64 * 1024 * 1024; + +struct QueryRequest { + std::uint32_t parameterCount; + std::string sql; +}; + +struct ValidationRequest { + std::vector schemaStatements; + std::vector queries; + std::vector migrationStatements; +}; + +class RequestReader { +public: + explicit RequestReader(std::string input) : input_(std::move(input)) {} + + bool readHeader(std::string* error) { + const std::size_t headerLength = std::strlen(kProtocolHeader); + if (input_.size() < headerLength || input_.compare(0, headerLength, kProtocolHeader) != 0) { + *error = "invalid request protocol header"; + return false; + } + offset_ = headerLength; + return true; + } + + bool readCount(std::uint32_t* value, std::string* error) { + if (!readUint32(value, error)) { + return false; + } + if (*value > kMaximumStatements) { + *error = "request contains too many statements"; + return false; + } + return true; + } + + bool readUint32(std::uint32_t* value, std::string* error) { + if (input_.size() - offset_ < 4) { + *error = "truncated uint32 in request"; + return false; + } + const auto* bytes = reinterpret_cast(input_.data() + offset_); + *value = (static_cast(bytes[0]) << 24) + | (static_cast(bytes[1]) << 16) + | (static_cast(bytes[2]) << 8) + | static_cast(bytes[3]); + offset_ += 4; + return true; + } + + bool readString(std::string* value, std::string* error) { + std::uint32_t length = 0; + if (!readUint32(&length, error)) { + return false; + } + if (length > kMaximumStatementBytes) { + *error = "SQL statement exceeds validator byte limit"; + return false; + } + if (input_.size() - offset_ < length) { + *error = "truncated SQL statement in request"; + return false; + } + value->assign(input_.data() + offset_, length); + offset_ += length; + if (value->find('\0') != std::string::npos) { + *error = "SQL statement contains an embedded NUL"; + return false; + } + return true; + } + + bool consumedAll(std::string* error) const { + if (offset_ != input_.size()) { + *error = "request contains trailing bytes"; + return false; + } + return true; + } + +private: + std::string input_; + std::size_t offset_ = 0; +}; + +std::string validatorIdentity() { + return std::string("valdi-clientsql-sqlite-validator protocol=1 sqlite=") + + kExpectedSQLiteVersion + " sqlite_source_id=" + kExpectedSQLiteSourceId + + " sqlite3_c_sha1=" + kExpectedSQLite3CSha1; +} + +std::string sanitizeError(std::string value) { + for (char& character : value) { + if (character == '\n' || character == '\r' || character == '\t') { + character = ' '; + } + } + return value; +} + +int reportError(const char* kind, std::size_t index, const std::string& message) { + std::cerr << "clientsql-validator-error:" << kind << ':' << index << ':' + << sanitizeError(message) << '\n'; + return 1; +} + +bool verifyLinkedSQLite(std::string* error) { + if (sqlite3_libversion_number() != kExpectedSQLiteVersionNumber + || std::strcmp(sqlite3_libversion(), kExpectedSQLiteVersion) != 0 + || std::strcmp(sqlite3_sourceid(), kExpectedSQLiteSourceId) != 0) { + std::ostringstream stream; + stream << "validator linked unexpected SQLite " << sqlite3_libversion() + << " source " << sqlite3_sourceid(); + *error = stream.str(); + return false; + } + return true; +} + +bool readRequest(ValidationRequest* request, std::string* error) { + std::ostringstream inputStream; + inputStream << std::cin.rdbuf(); + std::string input = inputStream.str(); + if (input.size() > kMaximumRequestBytes) { + *error = "validation request exceeds byte limit"; + return false; + } + + RequestReader reader(std::move(input)); + if (!reader.readHeader(error)) { + return false; + } + + std::uint32_t schemaCount = 0; + if (!reader.readCount(&schemaCount, error)) { + return false; + } + request->schemaStatements.reserve(schemaCount); + for (std::uint32_t index = 0; index < schemaCount; ++index) { + std::string sql; + if (!reader.readString(&sql, error)) { + return false; + } + request->schemaStatements.push_back(std::move(sql)); + } + + std::uint32_t queryCount = 0; + if (!reader.readCount(&queryCount, error)) { + return false; + } + request->queries.reserve(queryCount); + for (std::uint32_t index = 0; index < queryCount; ++index) { + std::uint32_t parameterCount = 0; + std::string sql; + if (!reader.readUint32(¶meterCount, error) || !reader.readString(&sql, error)) { + return false; + } + request->queries.push_back({parameterCount, std::move(sql)}); + } + + std::uint32_t migrationCount = 0; + if (!reader.readCount(&migrationCount, error)) { + return false; + } + request->migrationStatements.reserve(migrationCount); + for (std::uint32_t index = 0; index < migrationCount; ++index) { + std::string sql; + if (!reader.readString(&sql, error)) { + return false; + } + request->migrationStatements.push_back(std::move(sql)); + } + return reader.consumedAll(error); +} + +bool prepareExactlyOne( + sqlite3* database, + const std::string& sql, + sqlite3_stmt** output, + std::string* error +) { + if (sql.empty()) { + *error = "empty SQL statement"; + return false; + } + if (sql.size() > static_cast(std::numeric_limits::max())) { + *error = "SQL statement exceeds SQLite's input limit"; + return false; + } + + const char* cursor = sql.data(); + const char* end = cursor + sql.size(); + sqlite3_stmt* prepared = nullptr; + while (cursor < end) { + sqlite3_stmt* candidate = nullptr; + const char* tail = nullptr; + const int remaining = static_cast(end - cursor); + const int result = sqlite3_prepare_v2(database, cursor, remaining, &candidate, &tail); + if (result != SQLITE_OK) { + if (prepared != nullptr) { + sqlite3_finalize(prepared); + } + *error = sqlite3_errmsg(database); + return false; + } + if (tail == nullptr || tail <= cursor) { + if (candidate != nullptr) { + sqlite3_finalize(candidate); + } + if (prepared != nullptr) { + sqlite3_finalize(prepared); + } + *error = "SQLite validator made no progress while preparing SQL"; + return false; + } + if (candidate != nullptr) { + if (prepared != nullptr) { + sqlite3_finalize(candidate); + sqlite3_finalize(prepared); + *error = "validation record contains multiple SQL statements"; + return false; + } + prepared = candidate; + } + cursor = tail; + } + if (prepared == nullptr) { + *error = "SQL record contains no statement"; + return false; + } + *output = prepared; + return true; +} + +bool executePrepared(sqlite3* database, sqlite3_stmt* statement, std::string* error) { + for (;;) { + const int result = sqlite3_step(statement); + if (result == SQLITE_ROW) { + continue; + } + if (result == SQLITE_DONE) { + return true; + } + *error = sqlite3_errmsg(database); + return false; + } +} + +bool startsWith(const std::string& value, const char* prefix) { + const std::size_t prefixLength = std::strlen(prefix); + return value.size() >= prefixLength && value.compare(0, prefixLength, prefix) == 0; +} + +bool isMigrationResolutionError(const std::string& error) { + // Migration sources describe older schemas, so resolving them against the + // declared current schema can legitimately miss a retired table or column. + // These messages mean SQLite 3.16 completed parsing and reached name/schema + // resolution. Syntax and floor-semantic errors remain hard failures; this + // is not a list of SQL features accepted or rejected by ClientSQL. + return startsWith(error, "no such table:") + || startsWith(error, "no such column:") + || startsWith(error, "no such index:") + || startsWith(error, "duplicate column name:") + || (startsWith(error, "table ") && error.find(" already exists") != std::string::npos) + || (startsWith(error, "index ") && error.find(" already exists") != std::string::npos); +} + +int validateRequest(const ValidationRequest& request) { + sqlite3* database = nullptr; + const int openResult = sqlite3_open_v2( + ":memory:", + &database, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, + nullptr + ); + if (openResult != SQLITE_OK || database == nullptr) { + const std::string message = database == nullptr ? "could not allocate SQLite database" : sqlite3_errmsg(database); + if (database != nullptr) { + sqlite3_close(database); + } + return reportError("protocol", 0, message); + } + + char* pragmaError = nullptr; + if (sqlite3_exec(database, "PRAGMA foreign_keys = ON", nullptr, nullptr, &pragmaError) != SQLITE_OK) { + const std::string message = pragmaError == nullptr ? sqlite3_errmsg(database) : pragmaError; + sqlite3_free(pragmaError); + sqlite3_close(database); + return reportError("protocol", 0, message); + } + + for (std::size_t index = 0; index < request.schemaStatements.size(); ++index) { + sqlite3_stmt* statement = nullptr; + std::string error; + if (!prepareExactlyOne(database, request.schemaStatements[index], &statement, &error)) { + sqlite3_close(database); + return reportError("schema", index, error); + } + const bool executed = executePrepared(database, statement, &error); + sqlite3_finalize(statement); + if (!executed) { + sqlite3_close(database); + return reportError("schema", index, error); + } + } + + for (std::size_t index = 0; index < request.queries.size(); ++index) { + const QueryRequest& query = request.queries[index]; + sqlite3_stmt* statement = nullptr; + std::string error; + const std::string explainedSql = "EXPLAIN " + query.sql; + if (!prepareExactlyOne(database, explainedSql, &statement, &error)) { + sqlite3_close(database); + return reportError("query", index, error); + } + const int parameterCount = sqlite3_bind_parameter_count(statement); + if (parameterCount < 0 || static_cast(parameterCount) != query.parameterCount) { + std::ostringstream stream; + stream << "expected " << query.parameterCount << " parameters but SQLite prepared " << parameterCount; + sqlite3_finalize(statement); + sqlite3_close(database); + return reportError("query", index, stream.str()); + } + for (int parameter = 1; parameter <= parameterCount; ++parameter) { + if (sqlite3_bind_null(statement, parameter) != SQLITE_OK) { + error = sqlite3_errmsg(database); + sqlite3_finalize(statement); + sqlite3_close(database); + return reportError("query", index, error); + } + } + const bool executed = executePrepared(database, statement, &error); + sqlite3_finalize(statement); + if (!executed) { + sqlite3_close(database); + return reportError("query", index, error); + } + } + + for (std::size_t index = 0; index < request.migrationStatements.size(); ++index) { + sqlite3_stmt* statement = nullptr; + std::string error; + if (!prepareExactlyOne(database, request.migrationStatements[index], &statement, &error)) { + if (!isMigrationResolutionError(error)) { + sqlite3_close(database); + return reportError("migration", index, error); + } + continue; + } + sqlite3_finalize(statement); + } + + sqlite3_close(database); + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + std::string linkedSQLiteError; + if (!verifyLinkedSQLite(&linkedSQLiteError)) { + return reportError("identity", 0, linkedSQLiteError); + } + if (argc == 2 && std::strcmp(argv[1], "--version") == 0) { + std::cout << validatorIdentity() << '\n'; + return 0; + } + if (argc != 2 || std::strcmp(argv[1], "--validate") != 0) { + std::cerr << "usage: sqlite_316_validator --version|--validate\n"; + return 2; + } + + ValidationRequest request; + std::string requestError; + if (!readRequest(&request, &requestError)) { + return reportError("protocol", 0, requestError); + } + return validateRequest(request); +} diff --git a/compiler/clientsql/src/clientsql/cli.py b/compiler/clientsql/src/clientsql/cli.py index 9857ed2d8..5cd61150d 100644 --- a/compiler/clientsql/src/clientsql/cli.py +++ b/compiler/clientsql/src/clientsql/cli.py @@ -1,6 +1,8 @@ from __future__ import annotations import argparse +import hashlib +import pkgutil import sys from pathlib import Path from typing import Optional, Sequence @@ -13,36 +15,87 @@ load_type_mapping, parse_sql_file, sanitize_type_name, - validate_schema_and_queries, + validate_generated_identifiers, ) from .typescript import write_database_file, write_queries_file, write_types_file +from .validator import SQLite316Validator -VERSION = "valdi-clientsql 0.2.0" +SOURCE_FILES = ( + "__init__.py", + "__main__.py", + "cli.py", + "model.py", + "sql.py", + "typescript.py", + "validator.py", +) -def main(argv: Sequence[str]) -> int: - if "-version" in argv or "--version" in argv: - print(VERSION) - return 0 +def source_digest() -> str: + digest = hashlib.sha256() + for source_file in SOURCE_FILES: + source = pkgutil.get_data("clientsql", source_file) + if source is None: + raise RuntimeError(f"Missing ClientSQL generator source '{source_file}'") + encoded_name = source_file.encode("utf-8") + digest.update(len(encoded_name).to_bytes(4, "big")) + digest.update(encoded_name) + digest.update(len(source).to_bytes(8, "big")) + digest.update(source) + return digest.hexdigest() + + +def generator_version(validator: SQLite316Validator) -> str: + return ( + f"valdi-clientsql 0.2.0+source.sha256.{source_digest()}" + f"+validator.{validator.cache_identity}" + ) + +def main(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser(prog="clientsql") - parser.add_argument("-s", "--source", required=True, help="SQL source directory") - parser.add_argument("-p", "--package", required=True, help="Database package/name") - parser.add_argument("-c", "--class", dest="class_name", required=True, help="Database class name") - parser.add_argument("-m", "--module", required=True, help="Module name") - parser.add_argument("-o", "--output", required=True, help="Output directory") - parser.add_argument("-l", "--language", required=True, choices=["typescript"], help="Output language") + parser.add_argument("-version", "--version", action="store_true", help="Print generator identity") + parser.add_argument( + "--sqlite-validator", + help="Path to the hermetic SQLite 3.16.0 validation executable", + ) + parser.add_argument("-s", "--source", help="SQL source directory") + parser.add_argument("-p", "--package", help="Database package/name") + parser.add_argument("-c", "--class", dest="class_name", help="Database class name") + parser.add_argument("-m", "--module", help="Module name") + parser.add_argument("-o", "--output", help="Output directory") + parser.add_argument("-l", "--language", choices=["typescript"], help="Output language") parser.add_argument("-tm", "--type-mapping", dest="type_mapping", help="Optional sql_types.yaml") args = parser.parse_args(argv) try: + validator = SQLite316Validator.resolve(args.sqlite_validator) + if args.version: + print(generator_version(validator)) + return 0 + missing_arguments = [ + option + for option, value in ( + ("--source", args.source), + ("--package", args.package), + ("--class", args.class_name), + ("--module", args.module), + ("--output", args.output), + ("--language", args.language), + ) + if value is None + ] + if missing_arguments: + parser.error(f"the following arguments are required: {', '.join(missing_arguments)}") generate( sql_dir=Path(args.source), package_name=args.package, class_name=args.class_name, + module_name=args.module, output_dir=Path(args.output), type_mapping=args.type_mapping, + validator=validator, ) except ClientSqlError as exc: print(f"ClientSQL error: {exc}", file=sys.stderr) @@ -55,8 +108,10 @@ def generate( sql_dir: Path, package_name: str, class_name: str, + module_name: str, output_dir: Path, type_mapping: Optional[str], + validator: SQLite316Validator, ) -> None: package_dir = sql_dir / package_name if not package_dir.is_dir(): @@ -77,7 +132,22 @@ def generate( create_statements = collect_create_statements(sql_text_by_path.values()) migrations = collect_migrations(sql_dir) - validate_schema_and_queries(create_statements, sql_files) + query_validations = [ + ( + f"query {sql_file.rel_to_package}:{query.name}", + query.runtime_sql, + len(query.param_order), + ) + for sql_file in sql_files + for query in sql_file.queries + ] + migration_validations = [ + (f"migration {version} statement {index + 1}", statement) + for version, statements in migrations + for index, statement in enumerate(statements) + ] + validator.validate(create_statements, query_validations, migration_validations) + validate_generated_identifiers(sql_files, tables) for sql_file in sql_files: write_types_file(output_dir, sql_file, tables) @@ -87,6 +157,7 @@ def generate( output_dir=output_dir, class_name=sanitize_type_name(class_name), db_name=package_name, + module_name=module_name, sql_files=sql_files, create_statements=create_statements, migrations=migrations, diff --git a/compiler/clientsql/src/clientsql/sql.py b/compiler/clientsql/src/clientsql/sql.py index ab9f16caa..9e5ed5ff1 100644 --- a/compiler/clientsql/src/clientsql/sql.py +++ b/compiler/clientsql/src/clientsql/sql.py @@ -1,7 +1,6 @@ from __future__ import annotations import re -import sqlite3 import sys from pathlib import Path from typing import Dict, Iterable, List, Optional, Sequence, Tuple @@ -10,7 +9,15 @@ SQL_IDENTIFIER_PATTERN = r"[`\"\[]?[A-Za-z_][A-Za-z0-9_]*[`\"\]]?" - +MAX_INT32 = 2_147_483_647 +TYPESCRIPT_RESERVED_IDENTIFIERS = { + "break", "case", "catch", "class", "const", "constructor", "continue", "debugger", + "default", "delete", "do", "else", "enum", "export", "extends", "false", "finally", + "for", "function", "if", "implements", "import", "in", "instanceof", "interface", + "let", "new", "null", "package", "private", "protected", "public", "return", "static", + "super", "switch", "this", "throw", "true", "try", "typeof", "undefined", "var", "void", + "while", "with", "yield", +} def load_type_mapping(sql_dir: Path, type_mapping: Optional[str]) -> Dict[str, str]: if not type_mapping: @@ -95,6 +102,10 @@ def collect_migrations(sql_dir: Path) -> List[Tuple[int, List[str]]]: if version_match is None: raise ClientSqlError(f"Migration filename must start with a version number: {path}") version = int(version_match.group(1)) + if version < 2 or version > MAX_INT32: + raise ClientSqlError( + f"Migration version must be an integer from 2 through {MAX_INT32}: {path}" + ) if version in seen_versions: raise ClientSqlError(f"Duplicate migration version {version}: {path}") seen_versions.add(version) @@ -111,27 +122,92 @@ def collect_migrations(sql_dir: Path) -> List[Tuple[int, List[str]]]: return migrations -def validate_schema_and_queries(create_statements: Sequence[str], sql_files: Sequence[SqlFile]) -> None: - database = sqlite3.connect(":memory:") - try: - database.execute("PRAGMA foreign_keys = ON") - for statement in create_statements: - try: - database.execute(statement) - except sqlite3.Error as exc: - raise ClientSqlError(f"Invalid schema statement: {exc}\n{statement}") from exc - - for sql_file in sql_files: - for query in sql_file.queries: - try: - database.execute(f"EXPLAIN {query.runtime_sql}", [None] * len(query.param_order)) - except sqlite3.Error as exc: +def validate_generated_identifiers(sql_files: Sequence[SqlFile], tables: Dict[str, Table]) -> None: + query_class_names: Dict[str, Path] = {} + query_property_names: Dict[str, Path] = {} + for sql_file in sql_files: + class_name = f"{sanitize_type_name(sql_file.stem_path.name)}Queries" + property_name = class_name.removesuffix("Queries") + property_name = property_name[:1].lower() + property_name[1:] + "Queries" + for generated_name, owner_map, kind in ( + (class_name, query_class_names, "query class"), + (property_name, query_property_names, "database query property"), + ): + previous = owner_map.get(generated_name) + if previous is not None: + raise ClientSqlError( + f"Generated {kind} identifier '{generated_name}' collides between " + f"{previous} and {sql_file.rel_to_package}" + ) + owner_map[generated_name] = sql_file.rel_to_package + + generated_methods: Dict[str, str] = {} + emitted_symbols: Dict[str, str] = {} + for table in tables.values(): + generated_name = sanitize_type_name(table.name) + owner = f"table '{table.name}'" + previous = emitted_symbols.get(generated_name) + if previous is not None and previous != owner: + raise ClientSqlError( + f"Generated type identifier '{generated_name}' collides between {previous} and {owner} " + f"in {sql_file.rel_to_package}" + ) + emitted_symbols[generated_name] = owner + for query in sql_file.queries: + generated_parameters: Dict[str, str] = {} + for parameter_name in query.param_order: + generated_parameter = sanitize_identifier(parameter_name) + previous_parameter = generated_parameters.get(generated_parameter) + if previous_parameter is not None and previous_parameter != parameter_name: + raise ClientSqlError( + f"Generated query parameter identifier '{generated_parameter}' collides between " + f"'{previous_parameter}' and '{parameter_name}' in query '{query.name}' " + f"in {sql_file.rel_to_package}" + ) + generated_parameters[generated_parameter] = parameter_name + + method_name = sanitize_identifier(query.name) + method_names = [method_name] + if query.returns_rows: + method_names.append(f"watch{sanitize_type_name(query.name)}") + for generated_name in method_names: + previous = generated_methods.get(generated_name) + if previous is not None: + raise ClientSqlError( + f"Generated query method identifier '{generated_name}' collides between " + f"'{previous}' and '{query.name}' in {sql_file.rel_to_package}" + ) + generated_methods[generated_name] = query.name + + for generated_name in ( + f"{sanitize_type_name(query.name)}Params" if query.params else "", + query.result_type + if query.returns_rows and query.result_type == f"{sanitize_type_name(query.name)}Row" + else "", + ): + if not generated_name: + continue + owner = ( + f"query parameters for '{query.name}'" + if generated_name.endswith("Params") + else f"query row for '{query.name}'" + ) + previous = emitted_symbols.get(generated_name) + if previous is not None and previous != owner: raise ClientSqlError( - f"Invalid query {sql_file.rel_to_package}:{query.name}: {exc}" - ) from exc - finally: - database.close() + f"Generated type identifier '{generated_name}' collides between " + f"{previous} and {owner} in {sql_file.rel_to_package}" + ) + emitted_symbols[generated_name] = owner + field_names: set[str] = set() + for field in query.result_fields: + if field.name in field_names: + raise ClientSqlError( + f"Duplicate result field '{field.name}' in query '{query.name}' " + f"in {sql_file.rel_to_package}" + ) + field_names.add(field.name) def migration_sort_key(path: Path) -> Tuple[int, str]: match = re.match(r"(\d+)", path.stem) @@ -238,45 +314,15 @@ def analyze_query(name: str, sql: str, tables: Dict[str, Table]) -> Query: def normalize_params(sql: str) -> Tuple[str, List[str]]: + occurrences = scan_param_occurrences(sql) out: List[str] = [] - params: List[str] = [] - index = 0 - i = 0 - quote: Optional[str] = None - while i < len(sql): - ch = sql[i] - if quote: - out.append(ch) - if ch == quote: - if i + 1 < len(sql) and sql[i + 1] == quote: - out.append(sql[i + 1]) - i += 2 - continue - quote = None - i += 1 - continue - if ch in {"'", '"'}: - quote = ch - out.append(ch) - i += 1 - continue - if ch == "?": - param_name = f"p{index}" - params.append(param_name) - out.append("?") - index += 1 - i += 1 - continue - if ch == ":" and i + 1 < len(sql) and re.match(r"[A-Za-z_]", sql[i + 1]): - match = re.match(r":([A-Za-z_][A-Za-z0-9_]*)(\?)?", sql[i:]) - if match: - params.append(match.group(1)) - out.append("?") - i += len(match.group(0)) - continue - out.append(ch) - i += 1 - return "".join(out), params + cursor = 0 + for occurrence in occurrences: + out.append(sql[cursor:occurrence.start]) + out.append("?") + cursor = occurrence.end + out.append(sql[cursor:]) + return "".join(out), [occurrence.name for occurrence in occurrences] def infer_params(sql: str, param_order: List[str], tables: Dict[str, Table]) -> List[Parameter]: @@ -413,15 +459,10 @@ def infer_limit_param_types(sql: str, param_order: List[str]) -> Dict[str, str]: def infer_nullable_params(sql: str, param_order: List[str]) -> set[str]: occurrences = scan_param_occurrences(sql) nullable = {occurrence.name for occurrence in occurrences if occurrence.nullable} - nullable.update( - match.group(1) - for match in re.finditer(r":([A-Za-z_][A-Za-z0-9_]*)\s+IS\s+(?:NOT\s+)?NULL\b", sql, re.IGNORECASE) - ) - - for match in re.finditer(r"\?\s+IS\s+(?:NOT\s+)?NULL\b", sql, re.IGNORECASE): - param_name = param_name_at(sql, match.start(), param_order) - if param_name is not None: - nullable.add(param_name) + code = mask_sql_non_code(sql) + for occurrence in occurrences: + if re.match(r"\s+IS\s+(?:NOT\s+)?NULL\b", code[occurrence.end:], re.IGNORECASE): + nullable.add(occurrence.name) return nullable @@ -445,42 +486,79 @@ def param_name_at(sql: str, start: int, param_order: List[str]) -> Optional[str] def scan_param_occurrences(sql: str) -> List[ParamOccurrence]: occurrences: List[ParamOccurrence] = [] positional_index = 0 - i = 0 - quote: Optional[str] = None - while i < len(sql): - ch = sql[i] - if quote: - if ch == quote: - if i + 1 < len(sql) and sql[i + 1] == quote: - i += 2 + for start, end in sql_code_ranges(sql): + i = start + while i < end: + ch = sql[i] + if ch == "?": + occurrences.append( + ParamOccurrence(start=i, end=i + 1, name=f"p{positional_index}", nullable=False) + ) + positional_index += 1 + i += 1 + continue + if ch == ":" and i + 1 < end and re.match(r"[A-Za-z_]", sql[i + 1]): + match = re.match(r":([A-Za-z_][A-Za-z0-9_]*)(\?)?", sql[i:end]) + if match: + occurrences.append( + ParamOccurrence( + start=i, + end=i + len(match.group(0)), + name=match.group(1), + nullable=bool(match.group(2)), + ) + ) + i += len(match.group(0)) continue - quote = None i += 1 - continue - if ch in {"'", '"'}: - quote = ch + return occurrences + + +def sql_code_ranges(sql: str) -> List[Tuple[int, int]]: + """Return code spans, excluding literals, quoted identifiers, and comments.""" + ranges: List[Tuple[int, int]] = [] + code_start = 0 + i = 0 + while i < len(sql): + quote = sql[i] + is_line_comment = sql.startswith("--", i) + is_block_comment = sql.startswith("/*", i) + is_quoted = quote in {"'", '"', "`", "["} + if not is_line_comment and not is_block_comment and not is_quoted: i += 1 continue - if ch == "?": - occurrences.append(ParamOccurrence(start=i, end=i + 1, name=f"p{positional_index}", nullable=False)) - positional_index += 1 + + if code_start < i: + ranges.append((code_start, i)) + if is_line_comment: + newline = sql.find("\n", i + 2) + i = len(sql) if newline == -1 else newline + 1 + elif is_block_comment: + terminator = sql.find("*/", i + 2) + i = len(sql) if terminator == -1 else terminator + 2 + else: + closing_quote = "]" if quote == "[" else quote i += 1 - continue - if ch == ":" and i + 1 < len(sql) and re.match(r"[A-Za-z_]", sql[i + 1]): - match = re.match(r":([A-Za-z_][A-Za-z0-9_]*)(\?)?", sql[i:]) - if match: - occurrences.append( - ParamOccurrence( - start=i, - end=i + len(match.group(0)), - name=match.group(1), - nullable=bool(match.group(2)), - ) - ) - i += len(match.group(0)) - continue - i += 1 - return occurrences + while i < len(sql): + if sql[i] != closing_quote: + i += 1 + continue + if i + 1 < len(sql) and sql[i + 1] == closing_quote: + i += 2 + continue + i += 1 + break + code_start = i + if code_start < len(sql): + ranges.append((code_start, len(sql))) + return ranges + + +def mask_sql_non_code(sql: str) -> str: + masked = [" " if not character.isspace() else character for character in sql] + for start, end in sql_code_ranges(sql): + masked[start:end] = sql[start:end] + return "".join(masked) def infer_result_fields(sql: str, tables: Dict[str, Table]) -> List[Column]: @@ -667,6 +745,8 @@ def sanitize_identifier(name: str) -> str: cleaned = re.sub(r"\W+", "_", name) if not cleaned or re.match(r"\d", cleaned): cleaned = f"p_{cleaned}" + if cleaned in TYPESCRIPT_RESERVED_IDENTIFIERS: + cleaned = f"_{cleaned}" return cleaned diff --git a/compiler/clientsql/src/clientsql/typescript.py b/compiler/clientsql/src/clientsql/typescript.py index 4ae878a41..3cf3362b8 100644 --- a/compiler/clientsql/src/clientsql/typescript.py +++ b/compiler/clientsql/src/clientsql/typescript.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os import re @@ -62,8 +63,12 @@ def render_queries(sql_file: SqlFile, declare: bool) -> str: "export type ClientSQLQueryListener = (value: T) => void;", "", "export interface ClientSQLDatabase {", - " execute(sql: string, parameters?: ClientSQLValue[], changedTables?: string[]): Promise;", - " query(sql: string, parameters?: ClientSQLValue[]): Promise;", + " execute(", + " sql: string,", + " parameters: ClientSQLValue[] | undefined,", + " changedTables: string[] | undefined,", + " ): Promise;", + " query(sql: string, parameters: ClientSQLValue[] | undefined): Promise;", " watchQuery(", " tables: string[],", " load: () => Promise,", @@ -72,12 +77,32 @@ def render_queries(sql_file: SqlFile, declare: bool) -> str: "}", "", ]) + if any(boolean_result_fields(query) for query in sql_file.queries): + lines.extend([ + "function decodeClientSQLBoolean(value: ClientSQLValue | undefined, field: string): boolean {", + " if (value === true || value === 1) {", + " return true;", + " }", + " if (value === false || value === 0) {", + " return false;", + " }", + " throw new Error(`ClientSQL boolean field '${field}' returned invalid value '${String(value)}'`);", + "}", + "", + "function decodeNullableClientSQLBoolean(", + " value: ClientSQLValue | undefined,", + " field: string,", + "): boolean | null {", + " return value === null ? null : decodeClientSQLBoolean(value, field);", + "}", + "", + ]) if declare: lines.append(f"export declare class {class_name} {{") lines.append(" constructor(db: ClientSQLDatabase);") for query in sql_file.queries: - lines.append(f" {query.name}({method_signature_params(query)}): {method_return_type(query)};") + lines.append(f" {sanitize_identifier(query.name)}({method_signature_params(query)}): {method_return_type(query)};") if query.returns_rows: lines.append(f" {watch_method_name(query)}({watch_method_signature_params(query)}): ClientSQLSubscription;") lines.append("}") @@ -99,14 +124,26 @@ def render_query_method(query: Query) -> List[str]: sql_literal = json.dumps(query.runtime_sql) param_array = ", ".join(sanitize_identifier(name) for name in query.param_order) lines = [ - f" {query.name}({method_signature_params(query)}): {method_return_type(query)} {{", + f" {sanitize_identifier(query.name)}({method_signature_params(query)}): {method_return_type(query)} {{", ] if query.returns_rows: - lines.append(f" return this.db.query<{query.result_type}>({sql_literal}, [{param_array}]);") + boolean_fields = boolean_result_fields(query) + if boolean_fields: + lines.append( + f" return this.db.query>({sql_literal}, [{param_array}]).then(rows => rows.map(row => ({{" + ) + lines.append(" ...row,") + for field in boolean_fields: + decoder = "decodeNullableClientSQLBoolean" if field.nullable else "decodeClientSQLBoolean" + field_name = json.dumps(field.name) + lines.append(f" {render_property_name(field.name)}: {decoder}(row[{field_name}], {field_name}),") + lines.append(f" }}) as unknown as {query.result_type}));") + else: + lines.append(f" return this.db.query<{query.result_type}>({sql_literal}, [{param_array}]);") elif query.changed_tables: lines.append(f" return this.db.execute({sql_literal}, [{param_array}], {json.dumps(query.changed_tables)});") else: - lines.append(f" return this.db.execute({sql_literal}, [{param_array}]);") + lines.append(f" return this.db.execute({sql_literal}, [{param_array}], undefined);") lines.append(" }") return lines @@ -114,7 +151,8 @@ def render_query_method(query: Query) -> List[str]: def render_query_watch_method(query: Query) -> List[str]: tables_literal = json.dumps(query.read_tables) param_values = ", ".join(param.name for param in query.params) - invocation = f"this.{query.name}({param_values})" if param_values else f"this.{query.name}()" + method_name = sanitize_identifier(query.name) + invocation = f"this.{method_name}({param_values})" if param_values else f"this.{method_name}()" return [ f" {watch_method_name(query)}({watch_method_signature_params(query)}): ClientSQLSubscription {{", f" return this.db.watchQuery({tables_literal}, () => {invocation}, listener);", @@ -126,22 +164,36 @@ def write_database_file( output_dir: Path, class_name: str, db_name: str, + module_name: str, sql_files: List[SqlFile], create_statements: List[str], migrations: List[Tuple[int, List[str]]], ) -> None: path = output_dir / f"{class_name}.ts" - path.write_text(render_database(class_name, db_name, sql_files, create_statements, migrations, declare=False), encoding="utf-8") + path.write_text( + render_database( + class_name, + db_name, + module_name, + sql_files, + create_statements, + migrations, + declare=False, + ), + encoding="utf-8", + ) def render_database( class_name: str, db_name: str, + module_name: str, sql_files: List[SqlFile], create_statements: List[str], migrations: List[Tuple[int, List[str]]], declare: bool, ) -> str: + namespace_digest = hashlib.sha256(module_name.encode("utf-8")).hexdigest()[:24] query_classes = [(query_class_name(sql_file), import_path_from_db(sql_file, "Queries")) for sql_file in sql_files if sql_file.queries] all_tables = sorted({ table @@ -220,8 +272,12 @@ def render_database( property_name = lower_first(klass.removesuffix("Queries")) + "Queries" lines.append(f" readonly {property_name}: {klass};") lines.extend([ - " execute(sql: string, parameters?: ClientSQLValue[], changedTables?: string[]): Promise;", - " query(sql: string, parameters?: ClientSQLValue[]): Promise;", + " execute(", + " sql: string,", + " parameters: ClientSQLValue[] | undefined,", + " changedTables: string[] | undefined,", + " ): Promise;", + " query(sql: string, parameters: ClientSQLValue[] | undefined): Promise;", f" transaction(body: (transaction: {class_name}Transaction) => Promise): Promise;", "}", "", @@ -234,9 +290,13 @@ def render_database( for klass, _ in query_classes: property_name = lower_first(klass.removesuffix("Queries")) + "Queries" lines.append(f" readonly {property_name}: {klass};") - lines.append(" static open(name?: string): " + class_name + ";") - lines.append(" execute(sql: string, parameters?: ClientSQLValue[], changedTables?: string[]): Promise;") - lines.append(" query(sql: string, parameters?: ClientSQLValue[]): Promise;") + lines.append(" static open(name: string | undefined): " + class_name + ";") + lines.append(" execute(") + lines.append(" sql: string,") + lines.append(" parameters: ClientSQLValue[] | undefined,") + lines.append(" changedTables: string[] | undefined,") + lines.append(" ): Promise;") + lines.append(" query(sql: string, parameters: ClientSQLValue[] | undefined): Promise;") lines.append(" watchQuery(") lines.append(" tables: string[],") lines.append(" load: () => Promise,") @@ -252,6 +312,7 @@ def render_database( schema_version = max([version for version, _ in migrations], default=1) lines.extend([ f"const DEFAULT_DATABASE_NAME = {json.dumps(db_name)};", + f"const DATABASE_NAMESPACE = {json.dumps(namespace_digest)};", f"const SCHEMA_VERSION = {schema_version};", f"const CREATE_STATEMENTS: string[] = {json.dumps(create_statements, indent=2)};", f"const MIGRATIONS: ClientSQLMigration[] = {json.dumps([{'version': version, 'statements': statements} for version, statements in migrations], indent=2)};", @@ -276,6 +337,21 @@ def render_database( " return clientSQLNativeForTests ?? (require('client_sql/src/ClientSQLNative') as ClientSQLNativeModule);", "}", "", + "function databaseIdentity(name: string): string {", + " if (!name || name.length > 48) {", + " throw new Error('ClientSQL database names must contain from 1 through 48 UTF-16 code units');", + " }", + " let encodedName = '';", + " for (let index = 0; index < name.length; index += 1) {", + " const codeUnit = name.charCodeAt(index);", + " if (codeUnit < 0x20 || (codeUnit >= 0x7f && codeUnit <= 0x9f)) {", + " throw new Error('ClientSQL database names cannot contain control characters');", + " }", + " encodedName += codeUnit.toString(16).padStart(4, '0');", + " }", + " return `clientsql_${DATABASE_NAMESPACE}_${encodedName}.sqlite`;", + "}", + "", "function entriesForDatabase(name: string): ClientSQLWatchEntry[] {", " let entries = watchEntriesByDatabaseName[name];", " if (!entries) {", @@ -376,6 +452,7 @@ def render_database( "", " private constructor(", " private readonly databaseName: string,", + " private readonly databaseIdentity: string,", " private readonly connection: ClientSQLNativeConnection,", " ) {", ]) @@ -383,15 +460,17 @@ def render_database( property_name = lower_first(klass.removesuffix("Queries")) + "Queries" lines.append(f" this.{property_name} = new {klass}(this);") lines.extend([ - " this.debugDatabase = {", + " const debugDatabase: ClientSQLDebugDatabase = {", + " id: this.databaseIdentity,", " name: this.databaseName,", " schemaVersion: SCHEMA_VERSION,", " createStatements: CREATE_STATEMENTS,", " migrations: MIGRATIONS,", - " query: (sql: string, parameters: ClientSQLValue[] = []): Promise => this.query(sql, parameters),", + " query: (sql: string, parameters: ClientSQLValue[] | undefined): Promise => this.query(sql, parameters),", " debugInfo: (): Promise> => this.debugInfo(),", " };", - " registerClientSQLDebugDatabase(this.debugDatabase);", + " this.debugDatabase = debugDatabase;", + " registerClientSQLDebugDatabase(debugDatabase);", ]) lines.append(" }") lines.append("") @@ -401,25 +480,37 @@ def render_database( if query_classes: lines.append("") lines.extend([ - f" static open(name: string = DEFAULT_DATABASE_NAME): {class_name} {{", - " const connection = getClientSQLNative().openDatabase(name, SCHEMA_VERSION, CREATE_STATEMENTS, MIGRATIONS);", - f" return new {class_name}(name, connection);", + f" static open(name: string | undefined): {class_name} {{", + " const databaseName = name ?? DEFAULT_DATABASE_NAME;", + " const identity = databaseIdentity(databaseName);", + " const connection = getClientSQLNative().openDatabase(identity, SCHEMA_VERSION, CREATE_STATEMENTS, MIGRATIONS);", + f" return new {class_name}(databaseName, identity, connection);", " }", "", - " async execute(sql: string, parameters: ClientSQLValue[] = [], changedTables?: string[]): Promise {", + " async execute(", + " sql: string,", + " parameters: ClientSQLValue[] | undefined,", + " changedTables: string[] | undefined,", + " ): Promise {", " if (this.closed) {", " throw new Error(`ClientSQL database '${this.databaseName}' is closed`);", " }", - " return enqueueDatabaseWrite(this.databaseName, async () => {", + " if (this.activeTransactionCount > 0) {", + " throw new Error(`ClientSQL database '${this.databaseName}' cannot execute through its parent handle inside a transaction body`);", + " }", + " return enqueueDatabaseWrite(this.databaseIdentity, async () => {", " await nativePromise(callback => this.connection.execute(sql, parameters, callback));", " this.notifyTablesChanged(changedTables ?? ALL_TABLES);", " });", " }", "", - " async query(sql: string, parameters: ClientSQLValue[] = []): Promise {", + " async query(sql: string, parameters: ClientSQLValue[] | undefined): Promise {", " if (this.closed) {", " throw new Error(`ClientSQL database '${this.databaseName}' is closed`);", " }", + " if (this.activeTransactionCount > 0) {", + " throw new Error(`ClientSQL database '${this.databaseName}' cannot query through its parent handle inside a transaction body`);", + " }", " return nativePromise(callback => this.connection.query(sql, parameters, callback));", " }", "", @@ -454,14 +545,14 @@ def render_database( " return;", " }", " active = false;", - " const entries = watchEntriesByDatabaseName[this.databaseName];", + " const entries = watchEntriesByDatabaseName[this.databaseIdentity];", " if (entries) {", " removeWatchEntry(entries, entry);", " }", " removeWatchEntry(this.localWatchEntries, entry);", " };", " const entry: ClientSQLWatchEntry = { tables, emit, unsubscribe };", - " entriesForDatabase(this.databaseName).push(entry);", + " entriesForDatabase(this.databaseIdentity).push(entry);", " this.localWatchEntries.push(entry);", " emit();", "", @@ -474,12 +565,16 @@ def render_database( " if (this.closed) {", " throw new Error(`ClientSQL database '${this.databaseName}' is closed`);", " }", - " return enqueueDatabaseWrite(this.databaseName, () => this.runTransaction(body));", + " if (this.activeTransactionCount > 0) {", + " throw new Error(`ClientSQL database '${this.databaseName}' cannot start a parent transaction inside a transaction body`);", + " }", + " return enqueueDatabaseWrite(this.databaseIdentity, () => this.runTransaction(body));", " }", "", f" private async runTransaction(body: (transaction: {class_name}Transaction) => Promise): Promise {{", " let result!: T;", " const changedTables: string[] = [];", + " let committed = false;", " const transactionDebugId = this.nextTransactionDebugId++;", " const transactionStartedAtMs = Date.now();", " this.activeTransactionCount += 1;", @@ -518,7 +613,7 @@ def render_database( " changedTables: changedTables.slice(),", " changedTableCount: changedTables.length,", " });", - " this.emitChangedTables(changedTables);", + " committed = true;", " return result;", " } catch (error) {", " const transactionCompletedAtMs = Date.now();", @@ -532,17 +627,24 @@ def render_database( " changedTableCount: changedTables.length,", " error: errorMessage(error),", " });", - " notifyClientSQLDebugChanged(this.databaseName);", + " notifyClientSQLDebugChanged(this.databaseIdentity);", " throw error;", " } finally {", " this.activeTransactionChangedTables = undefined;", " this.activeTransactionCount -= 1;", + " if (committed) {", + " this.emitChangedTables(changedTables);", + " }", " }", " }", "", f" private createTransactionScope(nativeTransaction: ClientSQLNativeTransaction, changedTables: string[]): {class_name}Transaction {{", " const transactionDatabase = {", - " execute: async (sql: string, parameters: ClientSQLValue[] = [], tables?: string[]): Promise => {", + " execute: async (", + " sql: string,", + " parameters: ClientSQLValue[] | undefined,", + " tables: string[] | undefined,", + " ): Promise => {", " await nativePromise(callback => nativeTransaction.execute(sql, parameters, callback));", " const invalidatedTables = tables ?? ALL_TABLES;", " invalidatedTables.forEach(table => {", @@ -551,7 +653,7 @@ def render_database( " }", " });", " },", - " query: (sql: string, parameters: ClientSQLValue[] = []): Promise =>", + " query: (sql: string, parameters: ClientSQLValue[] | undefined): Promise =>", " nativePromise(callback => nativeTransaction.query(sql, parameters, callback)),", " watchQuery: (", " _tables: string[],", @@ -579,13 +681,16 @@ def render_database( " if (this.closed) {", " return;", " }", + " if (this.activeTransactionCount > 0) {", + " throw new Error(`ClientSQL database '${this.databaseName}' cannot close through its parent handle inside a transaction body`);", + " }", " this.closed = true;", " this.localWatchEntries.slice().forEach(entry => {", " if (typeof entry.unsubscribe === 'function') {", " entry.unsubscribe();", " return;", " }", - " const entries = watchEntriesByDatabaseName[this.databaseName];", + " const entries = watchEntriesByDatabaseName[this.databaseIdentity];", " if (entries) {", " removeWatchEntry(entries, entry);", " }", @@ -595,7 +700,7 @@ def render_database( " unregisterClientSQLDebugDatabase(this.debugDatabase);", " this.debugDatabase = undefined;", " }", - " await enqueueDatabaseWrite(this.databaseName, () => nativePromise(callback => this.connection.close(callback)));", + " await enqueueDatabaseWrite(this.databaseIdentity, () => nativePromise(callback => this.connection.close(callback)));", " }", "", " private async debugInfo(): Promise> {", @@ -613,9 +718,9 @@ def render_database( " pendingChangedTableCount: this.activeTransactionChangedTables?.length ?? 0,", " transactionHistoryCount: this.transactionHistory.length,", " transactions: this.transactionHistory.slice().reverse(),", - " watcherCount: (watchEntriesByDatabaseName[this.databaseName] || []).length,", + " watcherCount: (watchEntriesByDatabaseName[this.databaseIdentity] || []).length,", " localWatcherCount: this.localWatchEntries.length,", - " queuedWrite: writeChainsByDatabaseName[this.databaseName] !== undefined,", + " queuedWrite: writeChainsByDatabaseName[this.databaseIdentity] !== undefined,", " };", " }", "", @@ -633,8 +738,8 @@ def render_database( " }", "", " private emitChangedTables(changedTables: string[]): void {", - " notifyClientSQLDebugChanged(this.databaseName);", - " const entries = watchEntriesByDatabaseName[this.databaseName];", + " notifyClientSQLDebugChanged(this.databaseIdentity);", + " const entries = watchEntriesByDatabaseName[this.databaseIdentity];", " if (!entries) {", " return;", " }", @@ -667,6 +772,14 @@ def types_imported_by_queries(queries: List[Query]) -> List[str]: return sorted(imports) +def boolean_result_fields(query: Query) -> List[object]: + return [ + field + for field in query.result_fields + if re.search(r"(?:^|\|\s*)boolean(?:\s*\||$)", field.ts_type) + ] + + def render_interface(name: str, fields: Sequence[object], declare_export: bool) -> List[str]: prefix = "export interface" if declare_export else "interface" lines = [f"{prefix} {name} {{"] diff --git a/compiler/clientsql/src/clientsql/validator.py b/compiler/clientsql/src/clientsql/validator.py new file mode 100644 index 000000000..1efdac628 --- /dev/null +++ b/compiler/clientsql/src/clientsql/validator.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import hashlib +import re +import struct +import subprocess +from pathlib import Path +from typing import Sequence, Tuple + +from .model import ClientSqlError + + +SQLITE_VALIDATOR_VERSION = "3.16.0" +SQLITE_VALIDATOR_SOURCE_ID = "2017-01-02 11:57:58 04ac0b75b1716541b2b97704f4809cb7ef19cccf" +SQLITE_VALIDATOR_SQLITE3_C_SHA1 = "e2920fb885569d14197c9b7958e6f1db573ee669" +SQLITE_VALIDATOR_PROTOCOL = 1 +SQLITE_VALIDATOR_DECLARED_IDENTITY = ( + "valdi-clientsql-sqlite-validator protocol=1 sqlite=3.16.0 " + f"sqlite_source_id={SQLITE_VALIDATOR_SOURCE_ID} " + f"sqlite3_c_sha1={SQLITE_VALIDATOR_SQLITE3_C_SHA1}" +) +PROTOCOL_HEADER = b"VALDI_CLIENTSQL_SQL_VALIDATOR_V1\n" +MAX_STATEMENT_BYTES = 16 * 1024 * 1024 +MAX_REQUEST_BYTES = 64 * 1024 * 1024 +VALIDATOR_ERROR_PATTERN = re.compile( + r"^clientsql-validator-error:([a-z]+):(\d+):(.*)$", + re.MULTILINE, +) + +QueryValidation = Tuple[str, str, int] +MigrationValidation = Tuple[str, str] + + +class SQLite316Validator: + def __init__(self, path: Path, binary_sha256: str): + self.path = path + self.binary_sha256 = binary_sha256 + + @property + def cache_identity(self) -> str: + source_revision = SQLITE_VALIDATOR_SOURCE_ID.rsplit(" ", 1)[-1] + return ( + f"sqlite-{SQLITE_VALIDATOR_VERSION}" + f".source-id-{source_revision}" + f".sqlite3-c-sha1-{SQLITE_VALIDATOR_SQLITE3_C_SHA1}" + f".protocol-{SQLITE_VALIDATOR_PROTOCOL}" + f".binary-sha256-{self.binary_sha256}" + ) + + @classmethod + def resolve(cls, path_value: str | None) -> "SQLite316Validator": + if not path_value: + raise ClientSqlError( + "ClientSQL requires --sqlite-validator pointing to the pinned SQLite 3.16.0 validator" + ) + path = Path(path_value).expanduser() + if not path.is_file(): + raise ClientSqlError(f"ClientSQL SQLite 3.16.0 validator does not exist: {path}") + try: + version_result = subprocess.run( + [str(path), "--version"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ClientSqlError( + f"ClientSQL could not execute SQLite 3.16.0 validator '{path}': {exc}" + ) from exc + declared_identity = version_result.stdout.strip() + if version_result.returncode != 0 or declared_identity != SQLITE_VALIDATOR_DECLARED_IDENTITY: + details = version_result.stderr.strip() or declared_identity or "no identity returned" + raise ClientSqlError( + "ClientSQL SQLite validator identity mismatch; expected exact SQLite 3.16.0 " + f"source {SQLITE_VALIDATOR_SOURCE_ID}, got: {details}" + ) + try: + binary_sha256 = cls._binary_sha256(path) + except OSError as exc: + raise ClientSqlError(f"ClientSQL could not hash SQLite validator '{path}': {exc}") from exc + return cls(path=path, binary_sha256=binary_sha256) + + def validate( + self, + schema_statements: Sequence[str], + queries: Sequence[QueryValidation], + migrations: Sequence[MigrationValidation], + ) -> None: + request = bytearray(PROTOCOL_HEADER) + self._append_uint32(request, len(schema_statements), "schema statement count") + for statement in schema_statements: + self._append_sql(request, statement) + self._append_uint32(request, len(queries), "query count") + for _context, sql, parameter_count in queries: + self._append_uint32(request, parameter_count, "query parameter count") + self._append_sql(request, sql) + self._append_uint32(request, len(migrations), "migration statement count") + for _context, statement in migrations: + self._append_sql(request, statement) + if len(request) > MAX_REQUEST_BYTES: + raise ClientSqlError( + f"ClientSQL SQL validation request exceeds the {MAX_REQUEST_BYTES}-byte limit" + ) + + try: + current_sha256 = self._binary_sha256(self.path) + except OSError as exc: + raise ClientSqlError(f"ClientSQL could not re-read SQLite validator '{self.path}': {exc}") from exc + if current_sha256 != self.binary_sha256: + raise ClientSqlError("ClientSQL SQLite 3.16.0 validator changed after identity resolution") + + try: + result = subprocess.run( + [str(self.path), "--validate"], + input=bytes(request), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ClientSqlError( + f"ClientSQL could not run pinned SQLite 3.16.0 validation: {exc}" + ) from exc + if result.returncode == 0: + if result.stdout: + raise ClientSqlError("ClientSQL SQLite 3.16.0 validator returned unexpected output") + return + + stderr = result.stderr.decode("utf-8", errors="replace") + match = VALIDATOR_ERROR_PATTERN.search(stderr) + if match is None: + raise ClientSqlError( + "ClientSQL SQLite 3.16.0 validator failed without a structured error: " + f"{stderr.strip() or f'exit {result.returncode}'}" + ) + kind = match.group(1) + index = int(match.group(2)) + message = match.group(3).strip() + if kind == "schema" and index < len(schema_statements): + context = f"schema statement {index + 1}" + elif kind == "query" and index < len(queries): + context = queries[index][0] + elif kind == "migration" and index < len(migrations): + context = migrations[index][0] + else: + context = f"validator {kind} record {index}" + raise ClientSqlError( + f"SQLite {SQLITE_VALIDATOR_VERSION} rejected {context}: {message}" + ) + + @staticmethod + def _append_uint32(output: bytearray, value: int, label: str) -> None: + if value < 0 or value > 0xFFFFFFFF: + raise ClientSqlError(f"ClientSQL {label} exceeds validator protocol range") + output.extend(struct.pack(">I", value)) + + @classmethod + def _append_sql(cls, output: bytearray, sql: str) -> None: + encoded = sql.encode("utf-8") + if b"\0" in encoded: + raise ClientSqlError("ClientSQL SQL cannot contain embedded NUL characters") + if len(encoded) > MAX_STATEMENT_BYTES: + raise ClientSqlError( + f"ClientSQL SQL exceeds the {MAX_STATEMENT_BYTES}-byte validator statement limit" + ) + cls._append_uint32(output, len(encoded), "SQL byte length") + output.extend(encoded) + + @staticmethod + def _binary_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as executable: + for chunk in iter(lambda: executable.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/compiler/clientsql/test_clientsql.py b/compiler/clientsql/test_clientsql.py index f0d44664b..8b87a1e02 100644 --- a/compiler/clientsql/test_clientsql.py +++ b/compiler/clientsql/test_clientsql.py @@ -1,4 +1,6 @@ import os +import re +import sqlite3 import subprocess import shutil import sys @@ -30,6 +32,7 @@ def environment_tool_path(name: str) -> Path | None: CLIENTSQL_PACKAGER = REPO_ROOT / "compiler" / "clientsql" / "package_clientsql.py" JAVASCRIPT_RUNNER = environment_tool_path("CLIENTSQL_TEST_JAVASCRIPT_RUNNER") TYPESCRIPT_COMPILER = environment_tool_path("CLIENTSQL_TEST_TYPESCRIPT_COMPILER") +SQLITE_VALIDATOR = environment_tool_path("CLIENTSQL_TEST_SQLITE_VALIDATOR") LOCAL_TYPESCRIPT_COMPILER = ( REPO_ROOT / "npm_modules" / "cli" / "node_modules" / "typescript" / "bin" / "tsc" ) @@ -44,6 +47,147 @@ def clientsql_command() -> List[str]: return [str(CLIENTSQL_EXECUTABLE)] return [sys.executable, str(CLIENTSQL_SOURCE)] + @classmethod + def setUpClass(cls) -> None: + if SQLITE_VALIDATOR is None or not SQLITE_VALIDATOR.is_file(): + raise RuntimeError( + "CLIENTSQL_TEST_SQLITE_VALIDATOR must name the built SQLite 3.16.0 validator" + ) + + @staticmethod + def validator_arguments() -> List[str]: + assert SQLITE_VALIDATOR is not None + return ["--sqlite-validator", str(SQLITE_VALIDATOR)] + + def test_native_contract_versions_every_exported_declaration(self) -> None: + native_contract = ( + REPO_ROOT + / "src" + / "valdi_modules" + / "src" + / "valdi" + / "client_sql" + / "src" + / "ClientSQLNative.d.ts" + ).read_text(encoding="utf-8") + exported_declarations = list(re.finditer(r"@(ExportModule|ExportModel|ExportProxy)\b", native_contract)) + self.assertEqual(4, len(exported_declarations)) + for index, declaration in enumerate(exported_declarations): + next_start = ( + exported_declarations[index + 1].start() + if index + 1 < len(exported_declarations) + else len(native_contract) + ) + declaration_block = native_contract[declaration.start():next_start] + self.assertEqual(1, declaration_block.count("@Version(__PLACEHOLDER__)")) + self.assertIn("Only this value boundary intentionally marshals as untyped", native_contract) + + def test_native_sqlite_dependency_shape_is_explicit_for_apple_and_default(self) -> None: + client_sql_build = ( + REPO_ROOT / "src" / "valdi_modules" / "src" / "valdi" / "client_sql" / "BUILD.bazel" + ).read_text(encoding="utf-8") + sqlite_build = (REPO_ROOT / "third-party" / "sqlite" / "sqlite.BUILD").read_text(encoding="utf-8") + sqlite_316_build = (REPO_ROOT / "third-party" / "sqlite" / "sqlite_316.BUILD").read_text(encoding="utf-8") + sqlite_module = (REPO_ROOT / "MODULE.bazel").read_text(encoding="utf-8") + sqlite_workspace = (REPO_ROOT / "bzl" / "dependencies.bzl").read_text(encoding="utf-8") + sqlite_inventory = (REPO_ROOT / "fossa-deps.yml").read_text(encoding="utf-8") + + self.assertIn('"//bzl/conditions:ios": ["-lsqlite3"]', client_sql_build) + self.assertIn('"//bzl/conditions:macos": ["-lsqlite3"]', client_sql_build) + self.assertIn('"//conditions:default": ["VALDI_CLIENTSQL_USE_BUNDLED_SQLITE"]', client_sql_build) + self.assertIn('"//conditions:default": ["@sqlite//:sqlite"]', client_sql_build) + self.assertNotIn("target_compatible_with", sqlite_build) + self.assertIn('name = "sqlite"', sqlite_316_build) + self.assertIn('"-DSQLITE_THREADSAFE=0"', sqlite_316_build) + for declaration in (sqlite_module, sqlite_workspace): + self.assertIn("sqlite-autoconf-3530400", declaration) + self.assertIn("0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c", declaration) + self.assertNotIn("sqlite-autoconf-3530100", declaration) + self.assertIn('name = "sqlite_316"', declaration) + self.assertIn("sqlite-amalgamation-3160000", declaration) + self.assertIn("3b5dfb65807e2b17e6463357df848e322badba01dc9a4a1de8fdbb72d448e3b0", declaration) + self.assertIn("version: 3.53.4", sqlite_inventory) + self.assertIn("sqlite-autoconf-3530400", sqlite_inventory) + + def test_real_runtime_integration_target_links_generated_smoke_and_native_factory(self) -> None: + valdi_build = (REPO_ROOT / "valdi" / "BUILD.bazel").read_text(encoding="utf-8") + smoke_source = ( + REPO_ROOT + / "valdi" + / "testdata" + / "resources" + / "modules" + / "client_sql_smoke" + / "src" + / "ClientSQLSmoke.ts" + ).read_text(encoding="utf-8") + integration_source = ( + REPO_ROOT / "valdi" / "test" / "integration" / "ClientSQLRuntime_tests.cpp" + ).read_text(encoding="utf-8") + + target = valdi_build[valdi_build.index('name = "test_client_sql_runtime_integration"'):] + self.assertIn("client_sql_smoke:client_sql_smoke_native_desktop", target) + self.assertIn('"test/integration/ClientSQLRuntime_tests.cpp"', target) + integration_function = smoke_source[smoke_source.index("export function runClientSQLNativeIntegration"):] + self.assertIn("TestDb.open(`runtime-integration-${Date.now()}`)", integration_function) + self.assertIn("new ArrayBuffer(0)", integration_function) + self.assertIn("rows[0].enabled !== true", integration_function) + self.assertNotIn("setClientSQLNativeForTests", integration_function) + self.assertIn("runClientSQLNativeIntegration", integration_source) + self.assertIn("client_sql_smoke/src/ClientSQLSmoke", integration_source) + + def test_source_generator_is_the_toolchain_executable(self) -> None: + toolchain = (REPO_ROOT / "bzl" / "valdi" / "BUILD.bazel").read_text(encoding="utf-8") + toolchain_contract = (REPO_ROOT / "bzl" / "valdi" / "valdi_toolchain.bzl").read_text(encoding="utf-8") + distributed_alias = (REPO_ROOT / "bin" / "BUILD.bazel").read_text(encoding="utf-8") + processor = ( + REPO_ROOT / "compiler" / "compiler" / "Compiler" / "Sources" / "Processors" / "ClientSqlProcessor.swift" + ).read_text(encoding="utf-8") + processor_test = ( + REPO_ROOT + / "compiler" + / "compiler" + / "Compiler" + / "Tests" + / "CompilerTests" + / "ClientSqlProcessorTests.swift" + ).read_text(encoding="utf-8") + compiler_invocation = ( + REPO_ROOT / "bzl" / "valdi" / "valdi_run_compiler.bzl" + ).read_text(encoding="utf-8") + validator_source = ( + REPO_ROOT / "compiler" / "clientsql" / "sqlite_316_validator.cpp" + ).read_text(encoding="utf-8") + sql_parser = ( + REPO_ROOT / "compiler" / "clientsql" / "src" / "clientsql" / "sql.py" + ).read_text(encoding="utf-8") + self.assertIn('sqldelight_compiler = "//compiler/clientsql:clientsql"', toolchain) + self.assertIn( + 'clientsql_sqlite_validator = "//compiler/clientsql:sqlite_316_validator"', + toolchain, + ) + self.assertRegex( + toolchain_contract, + r'"sqldelight_compiler": attr\.label\(\s+executable = True,\s+cfg = "exec"', + ) + self.assertIn('actual = "@valdi//compiler/clientsql:clientsql"', distributed_alias) + self.assertIn( + 'actual = "@valdi//compiler/clientsql:sqlite_316_validator"', + distributed_alias, + ) + self.assertNotRegex(distributed_alias, r'name = "sqldelight_compiler",\s+srcs = \[\]') + self.assertIn('metadata: ["artifact": output]', processor) + self.assertIn("moduleName: bundleInfo.name", processor) + self.assertIn('"--sqlite-validator", sqliteValidatorPath', processor) + self.assertIn('"--sqlite-validator", "/tools/sqlite_316_validator"', processor_test) + self.assertIn('args.add("--direct-client-sql-validator-path", client_sql_validator[0])', compiler_invocation) + self.assertIn('constexpr int kExpectedSQLiteVersionNumber = 3016000;', validator_source) + self.assertIn('e2920fb885569d14197c9b7958e6f1db573ee669', validator_source) + self.assertNotIn("import sqlite3", sql_parser) + self.assertNotIn("SQLITE_LATER_DIALECT_FEATURES", sql_parser) + self.assertIn('"-m", "MyBundle"', processor_test) + self.assertIn('package: "SharedDb"', processor_test) + def test_packaged_generator_matches_canonical_source(self) -> None: with tempfile.TemporaryDirectory() as directory: executable = Path(directory) / "clientsql" @@ -65,6 +209,71 @@ def test_packaged_generator_matches_canonical_source(self) -> None: ) self.assertEqual(check_result.returncode, 0, msg=check_result.stderr) + source_version = subprocess.run( + [*self.clientsql_command(), *self.validator_arguments(), "-version"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + packaged_version = subprocess.run( + [sys.executable, str(executable), *self.validator_arguments(), "-version"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(source_version.returncode, 0, msg=source_version.stderr) + self.assertEqual(packaged_version.returncode, 0, msg=packaged_version.stderr) + self.assertEqual(source_version.stdout, packaged_version.stdout) + self.assertRegex( + source_version.stdout, + r"0\.2\.0\+source\.sha256\.[0-9a-f]{64}" + r"\+validator\.sqlite-3\.16\.0\.source-id-[0-9a-f]{40}" + r"\.sqlite3-c-sha1-[0-9a-f]{40}\.protocol-1\.binary-sha256-[0-9a-f]{64}", + ) + + root = Path(directory) + sql_dir = root / "sql" + package_dir = sql_dir / "TestDb" + package_dir.mkdir(parents=True) + (package_dir / "Item.sq").write_text( + "CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY);\n\nselectAll:\nSELECT * FROM item;", + encoding="utf-8", + ) + source_output = root / "source-output" + packaged_output = root / "packaged-output" + common_arguments = [ + *self.validator_arguments(), + "-s", str(sql_dir), "-p", "TestDb", "-c", "TestDb", "-m", "FixtureModule", + "-l", "typescript", + ] + source_result = subprocess.run( + [*self.clientsql_command(), *common_arguments, "-o", str(source_output)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + packaged_result = subprocess.run( + [sys.executable, str(executable), *common_arguments, "-o", str(packaged_output)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(source_result.returncode, 0, msg=source_result.stderr) + self.assertEqual(packaged_result.returncode, 0, msg=packaged_result.stderr) + source_files = sorted(path.relative_to(source_output) for path in source_output.rglob("*.ts")) + packaged_files = sorted(path.relative_to(packaged_output) for path in packaged_output.rglob("*.ts")) + self.assertEqual(source_files, packaged_files) + for relative_path in source_files: + self.assertEqual( + (source_output / relative_path).read_bytes(), + (packaged_output / relative_path).read_bytes(), + msg=str(relative_path), + ) + def assert_in_order(self, text: str, *needles: str) -> None: cursor = 0 for needle in needles: @@ -76,6 +285,7 @@ def run_clientsql(self, sql_dir: Path, out_dir: Path) -> subprocess.CompletedPro return subprocess.run( [ *self.clientsql_command(), + *self.validator_arguments(), "-s", str(sql_dir), "-p", @@ -153,7 +363,7 @@ def run_generated_typescript(self, root: Path, entrypoint: Path) -> subprocess.C registeredDatabases.splice(index, 1); } } - export function notifyClientSQLDebugChanged(_databaseName?: string): void {} + export function notifyClientSQLDebugChanged(_databaseName: string | undefined): void {} """, encoding="utf-8", ) @@ -224,8 +434,12 @@ def test_generates_typescript_database_and_query_bindings(self) -> None: self.assertIn("export type ClientSQLNativeCallback", database) self.assertIn("client_sql/src/ClientSQLDebug", database) self.assertIn("private debugDatabase: ClientSQLDebugDatabase | undefined;", database) - self.assertIn("registerClientSQLDebugDatabase(this.debugDatabase);", database) + self.assertIn("const debugDatabase: ClientSQLDebugDatabase = {", database) + self.assertIn("this.debugDatabase = debugDatabase;", database) + self.assertIn("registerClientSQLDebugDatabase(debugDatabase);", database) self.assertIn("unregisterClientSQLDebugDatabase(this.debugDatabase);", database) + self.assertIn("id: this.databaseIdentity,", database) + self.assertIn("const identity = databaseIdentity(databaseName);", database) self.assertIn( "execute(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void;", database, @@ -243,7 +457,12 @@ def test_generates_typescript_database_and_query_bindings(self) -> None: database, ) self.assertIn("debugInfo: (): Promise> => this.debugInfo(),", database) - self.assertIn("changedTables?: string[]", database) + self.assertIn("changedTables: string[] | undefined", database) + self.assertIn("static open(name: string | undefined)", database) + self.assertNotIn("parameters?: ClientSQLValue[]", database) + self.assertNotIn("parameters: ClientSQLValue[] = []", database) + self.assertNotIn("changedTables?: string[]", database) + self.assertNotIn("name: string = DEFAULT_DATABASE_NAME", database) self.assertIn("export interface TestDbTransaction", database) self.assertIn("userQueries: new UserQueries(transactionDatabase)", database) self.assertIn("this.emitChangedTables(changedTables);", database) @@ -368,7 +587,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None self.assertIn("const chain = previous.then(() => current, () => current);", database) self.assertIn("writeChainsByDatabaseName[name] = chain;", database) self.assertIn("releaseDatabaseWrite(name, chain, releaseCurrent);", database) - self.assertIn("return enqueueDatabaseWrite(this.databaseName, async () => {", database) + self.assertIn("return enqueueDatabaseWrite(this.databaseIdentity, async () => {", database) self.assertIn("notifyClientSQLDebugChanged", database) self.assertIn("interface ClientSQLTransactionDebugEntry {", database) self.assertIn("const MAX_TRANSACTION_DEBUG_ENTRIES = 50;", database) @@ -376,7 +595,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None self.assertIn("private transactionHistory: ClientSQLTransactionDebugEntry[] = [];", database) self.assert_in_order( database, - "entriesForDatabase(this.databaseName).push(entry);", + "entriesForDatabase(this.databaseIdentity).push(entry);", "this.localWatchEntries.push(entry);", "emit();", ) @@ -397,7 +616,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None transaction = database[database.index(" async transaction") : database.index(" async close")] self.assert_in_order( transaction, - "return enqueueDatabaseWrite(this.databaseName, () => this.runTransaction(body));", + "return enqueueDatabaseWrite(this.databaseIdentity, () => this.runTransaction(body));", "private async runTransaction", "const changedTables: string[] = [];", ) @@ -423,7 +642,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None "this.recordTransactionDebugEntry({", "status: 'rolled_back',", "error: errorMessage(error),", - "notifyClientSQLDebugChanged(this.databaseName);", + "notifyClientSQLDebugChanged(this.databaseIdentity);", "throw error;", ) self.assertNotIn("BEGIN TRANSACTION", transaction) @@ -435,7 +654,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None self.assertIn("entry.unsubscribe();", close) self.assertIn("removeWatchEntry(this.localWatchEntries, entry);", close) self.assertIn( - "await enqueueDatabaseWrite(this.databaseName, () => nativePromise(callback => this.connection.close(callback)));", + "await enqueueDatabaseWrite(this.databaseIdentity, () => nativePromise(callback => this.connection.close(callback)));", close, ) @@ -444,7 +663,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None self.assertIn("pendingChangedTableCount: this.activeTransactionChangedTables?.length ?? 0,", debug) self.assertIn("transactionHistoryCount: this.transactionHistory.length,", debug) self.assertIn("transactions: this.transactionHistory.slice().reverse(),", debug) - self.assertIn("watcherCount: (watchEntriesByDatabaseName[this.databaseName] || []).length,", debug) + self.assertIn("watcherCount: (watchEntriesByDatabaseName[this.databaseIdentity] || []).length,", debug) recorder = database[database.index(" private recordTransactionDebugEntry") : database.index(" private notifyTablesChanged")] self.assertIn("this.transactionHistory.push(entry);", recorder) @@ -455,7 +674,7 @@ def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None self.assertNotIn("transactionDepth", notify) emit = database[database.index(" private emitChangedTables") : database.index("}", database.index(" private emitChangedTables"))] - self.assertIn("notifyClientSQLDebugChanged(this.databaseName);", emit) + self.assertIn("notifyClientSQLDebugChanged(this.databaseIdentity);", emit) def test_generated_watchers_execute_reactive_contract(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -624,6 +843,7 @@ class FakeConnection implements ClientSQLNativeConnection { class FakeNative implements ClientSQLNativeModule { private readonly storesByName: { [name: string]: Store | undefined } = Object.create(null); + lastOpenedName = ''; openDatabase( name: string, @@ -631,6 +851,7 @@ class FakeNative implements ClientSQLNativeModule { _createStatements: string[], _migrations: ClientSQLMigration[], ): ClientSQLNativeConnection { + this.lastOpenedName = name; return new FakeConnection(this.store(name)); } @@ -743,7 +964,8 @@ class FakeNative implements ClientSQLNativeModule { const native = new FakeNative(); setClientSQLNativeForTests(native); const first = TestDb.open('queue-contract'); - const store = native.store('queue-contract'); + const second = TestDb.open('queue-contract'); + const store = native.store(native.lastOpenedName); let releaseTransaction!: () => void; let markStarted!: () => void; const transactionStarted = new Promise(resolve => { @@ -755,6 +977,27 @@ class FakeNative implements ClientSQLNativeModule { const transactionPromise = first.transaction(async transaction => { await transaction.userQueries.insertUser(10, 'Inside'); + let parentExecuteRejected = false; + try { + await first.userQueries.insertUser(12, 'Reentrant'); + } catch (error) { + parentExecuteRejected = errorMessage(error).indexOf('parent handle inside a transaction body') !== -1; + } + assert(parentExecuteRejected, 'parent execute did not reject inside transaction body'); + let parentQueryRejected = false; + try { + await first.userQueries.selectAll(); + } catch (error) { + parentQueryRejected = errorMessage(error).indexOf('parent handle inside a transaction body') !== -1; + } + assert(parentQueryRejected, 'parent query did not reject inside transaction body'); + let parentCloseRejected = false; + try { + await first.close(); + } catch (error) { + parentCloseRejected = errorMessage(error).indexOf('parent handle inside a transaction body') !== -1; + } + assert(parentCloseRejected, 'parent close did not reject inside transaction body'); markStarted(); await transactionBlocker; }); @@ -762,7 +1005,7 @@ class FakeNative implements ClientSQLNativeModule { const activeDebugInfo = await (registeredDatabases[0] as any).debugInfo(); assert(activeDebugInfo.queuedWrite === true, 'active transaction was not reported as queued'); - const outsideWrite = first.userQueries.insertUser(11, 'Outside'); + const outsideWrite = second.userQueries.insertUser(11, 'Outside'); await nextTurn(); await nextTurn(); assert(!store.rows.some(row => row.id === 11), 'outside write interleaved into open transaction'); @@ -778,6 +1021,7 @@ class FakeNative implements ClientSQLNativeModule { const idleDebugInfo = await (registeredDatabases[0] as any).debugInfo(); assert(idleDebugInfo.queuedWrite === false, 'drained write queue remained marked active'); await first.close(); + await second.close(); } async function main(): Promise { @@ -832,6 +1076,327 @@ def test_nested_sql_files_preserve_output_paths(self) -> None: self.assertIn("selectRecent(limit: number): Promise", queries) self.assertIn("watchSelectRecent(limit: number, listener: ClientSQLQueryListener)", queries) + def test_generated_boolean_codecs_execute_nullable_true_and_false(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "Flag.sq").write_text( + """ + CREATE TABLE flag ( + id INTEGER NOT NULL PRIMARY KEY, + enabled BOOLEAN NOT NULL, + optional BOOLEAN + ); + + selectFlags: + SELECT enabled, optional FROM flag ORDER BY id; + """, + encoding="utf-8", + ) + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + self.assertEqual(result.returncode, 0, msg=result.stderr) + queries = (out_dir / "FlagQueries.ts").read_text(encoding="utf-8") + self.assertIn("function decodeClientSQLBoolean", queries) + self.assertIn("decodeNullableClientSQLBoolean", queries) + self.assertIn("enabled: decodeClientSQLBoolean", queries) + self.assertIn("optional: decodeNullableClientSQLBoolean", queries) + + entrypoint = root / "boolean_codec_test.ts" + entrypoint.write_text( + """ + import { ClientSQLDatabase, ClientSQLValue, FlagQueries } from './out/FlagQueries'; + + const db: ClientSQLDatabase = { + execute(_sql, _parameters, _changedTables): Promise { + return Promise.resolve(); + }, + query(_sql: string, _parameters: ClientSQLValue[] | undefined): Promise { + return Promise.resolve([ + { enabled: 0, optional: null }, + { enabled: 1, optional: 0 }, + { enabled: true, optional: 1 }, + ] as unknown as T[]); + }, + watchQuery(_tables: string[], _load: () => Promise, _listener: (value: T) => void) { + return { unsubscribe(): void {} }; + }, + }; + + void new FlagQueries(db).selectFlags().then(rows => { + if (rows[0].enabled !== false || rows[0].optional !== null) throw new Error('nullable false decode'); + if (rows[1].enabled !== true || rows[1].optional !== false) throw new Error('numeric decode'); + if (rows[2].enabled !== true || rows[2].optional !== true) throw new Error('boolean decode'); + }); + """, + encoding="utf-8", + ) + run_result = self.run_generated_typescript(root, entrypoint) + self.assertEqual( + run_result.returncode, + 0, + msg=f"stdout:\n{run_result.stdout}\nstderr:\n{run_result.stderr}", + ) + + def test_namespaces_database_identity_by_module_and_rejects_identifier_collisions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "Reserved.sq").write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY); + + default: + SELECT id FROM item WHERE id = :class; + """, + encoding="utf-8", + ) + + namespace_values = [] + for module_name in ["BundleOne", "BundleTwo"]: + out_dir = root / module_name + command = [ + *self.clientsql_command(), *self.validator_arguments(), + "-s", str(sql_dir), "-p", "TestDb", + "-c", "TestDb", "-m", module_name, "-o", str(out_dir), "-l", "typescript", + ] + result = subprocess.run(command, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertEqual(result.returncode, 0, msg=result.stderr) + database = (out_dir / "TestDb.ts").read_text(encoding="utf-8") + namespace_match = re.search(r'const DATABASE_NAMESPACE = "([0-9a-f]+)";', database) + self.assertIsNotNone(namespace_match) + namespace_values.append(namespace_match.group(1)) + queries = (out_dir / "ReservedQueries.ts").read_text(encoding="utf-8") + self.assertIn("_default(_class: number)", queries) + self.assertNotEqual(namespace_values[0], namespace_values[1]) + + (db_dir / "Reserved.sq").write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY); + + foo_bar: + SELECT id FROM item; + + fooBar: + SELECT id FROM item; + """, + encoding="utf-8", + ) + collision = self.run_clientsql(sql_dir, root / "collision") + self.assertNotEqual(collision.returncode, 0) + self.assertIn("collides between 'foo_bar' and 'fooBar'", collision.stderr) + + (db_dir / "Reserved.sq").write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY); + + selectCollision: + SELECT id FROM item WHERE id = :class OR id = :_class; + """, + encoding="utf-8", + ) + parameter_collision = self.run_clientsql(sql_dir, root / "parameter-collision") + self.assertNotEqual(parameter_collision.returncode, 0) + self.assertIn("query parameter identifier '_class' collides", parameter_collision.stderr) + + def test_rejects_table_row_and_parameter_type_symbol_collisions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + sql_file = db_dir / "Collision.sq" + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY); + CREATE TABLE SelectThingRow (id INTEGER NOT NULL PRIMARY KEY); + + selectThing: + SELECT id AS selected_id FROM item; + """, + encoding="utf-8", + ) + + row_collision = self.run_clientsql(sql_dir, root / "row-collision") + self.assertNotEqual(row_collision.returncode, 0) + self.assertIn("Generated type identifier 'SelectThingRow' collides", row_collision.stderr) + self.assertIn("table 'SelectThingRow'", row_collision.stderr) + self.assertIn("query row for 'selectThing'", row_collision.stderr) + + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY); + CREATE TABLE InsertThingParams (id INTEGER NOT NULL PRIMARY KEY); + + insertThing: + INSERT INTO item(id) VALUES (:id); + """, + encoding="utf-8", + ) + params_collision = self.run_clientsql(sql_dir, root / "params-collision") + self.assertNotEqual(params_collision.returncode, 0) + self.assertIn("Generated type identifier 'InsertThingParams' collides", params_collision.stderr) + self.assertIn("table 'InsertThingParams'", params_collision.stderr) + self.assertIn("query parameters for 'insertThing'", params_collision.stderr) + + def test_shared_sql_lexer_ignores_placeholders_in_literals_identifiers_and_comments(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "Lexer.sq").write_text( + """ + CREATE TABLE item ( + id INTEGER NOT NULL PRIMARY KEY, + note TEXT NOT NULL, + "id:quoted" TEXT, + `?` TEXT, + [:bracket] TEXT + ); + + selectLexer: + SELECT "id:quoted", `?`, [:bracket], note + FROM item + WHERE id = :id + AND note != ':literal ?' + -- :id IS NULL :line_comment ? + /* :id IS NULL :block_comment ? */; + """, + encoding="utf-8", + ) + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + self.assertEqual(result.returncode, 0, msg=result.stderr) + queries = (out_dir / "LexerQueries.ts").read_text(encoding="utf-8") + self.assertIn("selectLexer(id: number)", queries) + self.assertIn("-- :id IS NULL :line_comment ?", queries) + self.assertIn("/* :id IS NULL :block_comment ? */", queries) + self.assertNotIn("line_comment: ClientSQLValue", queries) + + def test_exact_sqlite_316_validator_rejects_later_syntax_and_accepts_supported_corpus(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + sql_file = db_dir / "Dialect.sq" + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL); + + insertReturning: + INSERT INTO item(id, note) VALUES (:id, :note) RETURNING id; + """, + encoding="utf-8", + ) + returning = self.run_clientsql(sql_dir, root / "returning") + self.assertNotEqual(returning.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected query Dialect.sq:insertReturning", returning.stderr) + + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL); + + ranked: + SELECT id, row_number() OVER (ORDER BY id) AS rank FROM item; + """, + encoding="utf-8", + ) + window = self.run_clientsql(sql_dir, root / "window") + self.assertNotEqual(window.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected query Dialect.sq:ranked", window.stderr) + + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL); + + aggregateWithoutGroup: + SELECT count(*) AS count FROM item HAVING count(*) > 0; + """, + encoding="utf-8", + ) + having = self.run_clientsql(sql_dir, root / "having") + self.assertNotEqual(having.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected query Dialect.sq:aggregateWithoutGroup", having.stderr) + self.assertIn("GROUP BY clause is required before HAVING", having.stderr) + if sqlite3.sqlite_version_info >= (3, 39, 0): + host = sqlite3.connect(":memory:") + try: + host.execute("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL)") + host.execute("SELECT count(*) FROM item HAVING count(*) > 0").fetchall() + finally: + host.close() + + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL) STRICT; + """, + encoding="utf-8", + ) + strict = self.run_clientsql(sql_dir, root / "strict") + self.assertNotEqual(strict.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected schema statement 1", strict.stderr) + + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL); + + upsert: + INSERT INTO item(id, note) VALUES (:id, :note) + ON CONFLICT(id) DO UPDATE SET note = excluded.note; + """, + encoding="utf-8", + ) + upsert = self.run_clientsql(sql_dir, root / "upsert") + self.assertNotEqual(upsert.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected query Dialect.sq:upsert", upsert.stderr) + + sql_file.write_text( + """ + CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, note TEXT NOT NULL); + + compatible: + WITH selected AS (SELECT id, note FROM item WHERE id = :id) + SELECT id, note FROM selected + GROUP BY id, note HAVING count(*) > 0; + """, + encoding="utf-8", + ) + compatible = self.run_clientsql(sql_dir, root / "compatible") + self.assertEqual(compatible.returncode, 0, msg=compatible.stderr) + + migration_dir = sql_dir / "migration" + migration_dir.mkdir() + (migration_dir / "2.sqm").write_text( + "SELECT count(*) FROM item HAVING count(*) > 0;", + encoding="utf-8", + ) + having_migration = self.run_clientsql(sql_dir, root / "having-migration") + self.assertNotEqual(having_migration.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected migration 2 statement 1", having_migration.stderr) + self.assertIn("GROUP BY clause is required before HAVING", having_migration.stderr) + + (migration_dir / "2.sqm").write_text( + "ALTER TABLE item DROP COLUMN note;", + encoding="utf-8", + ) + later_migration = self.run_clientsql(sql_dir, root / "later-migration") + self.assertNotEqual(later_migration.returncode, 0) + self.assertIn("SQLite 3.16.0 rejected migration 2 statement 1", later_migration.stderr) + + (migration_dir / "2.sqm").write_text( + "ALTER TABLE item ADD COLUMN created_at INTEGER;", + encoding="utf-8", + ) + compatible_migration = self.run_clientsql(sql_dir, root / "compatible-migration") + self.assertEqual(compatible_migration.returncode, 0, msg=compatible_migration.stderr) + def test_validates_sql_and_tracks_all_reactive_tables(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -885,7 +1450,7 @@ def test_validates_sql_and_tracks_all_reactive_tables(self) -> None: ) invalid_result = self.run_clientsql(sql_dir, root / "invalid-out") self.assertNotEqual(invalid_result.returncode, 0) - self.assertIn("Invalid query Feed.sq:invalidQuery", invalid_result.stderr) + self.assertIn("SQLite 3.16.0 rejected query Feed.sq:invalidQuery", invalid_result.stderr) self.assertIn("missing_column", invalid_result.stderr) def test_rejects_duplicate_query_and_migration_versions(self) -> None: @@ -931,9 +1496,21 @@ def test_rejects_duplicate_query_and_migration_versions(self) -> None: self.assertNotEqual(migration_gap.returncode, 0) self.assertIn("Migration versions must be contiguous starting at 2", migration_gap.stderr) + (migration_dir / "3.sqm").unlink() + (migration_dir / "0.sqm").write_text("SELECT 1;", encoding="utf-8") + zero_migration = self.run_clientsql(sql_dir, root / "migration-zero-out") + self.assertNotEqual(zero_migration.returncode, 0) + self.assertIn("Migration version must be an integer from 2", zero_migration.stderr) + + (migration_dir / "0.sqm").unlink() + (migration_dir / "2147483648.sqm").write_text("SELECT 1;", encoding="utf-8") + oversized_migration = self.run_clientsql(sql_dir, root / "migration-oversized-out") + self.assertNotEqual(oversized_migration.returncode, 0) + self.assertIn("Migration version must be an integer from 2", oversized_migration.stderr) + def test_version_contract(self) -> None: result = subprocess.run( - [*self.clientsql_command(), "-version"], + [*self.clientsql_command(), *self.validator_arguments(), "-version"], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -941,6 +1518,41 @@ def test_version_contract(self) -> None: ) self.assertEqual(result.returncode, 0) self.assertIn("valdi-clientsql", result.stdout) + self.assertRegex(result.stdout, r"source\.sha256\.[0-9a-f]{64}") + self.assertRegex(result.stdout, r"validator\.sqlite-3\.16\.0.*binary-sha256-[0-9a-f]{64}") + + def test_validator_resolution_fails_closed_when_missing_or_mismatched(self) -> None: + missing = subprocess.run( + [*self.clientsql_command(), "-version"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("requires --sqlite-validator", missing.stderr) + + with tempfile.TemporaryDirectory() as tmp: + fake_validator = Path(tmp) / "sqlite-validator" + fake_validator.write_text( + "#!/bin/sh\necho 'valdi-clientsql-sqlite-validator protocol=1 sqlite=3.49.2'\n", + encoding="utf-8", + ) + fake_validator.chmod(0o755) + mismatched = subprocess.run( + [ + *self.clientsql_command(), + "--sqlite-validator", + str(fake_validator), + "-version", + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(mismatched.returncode, 0) + self.assertIn("validator identity mismatch", mismatched.stderr) if __name__ == "__main__": diff --git a/compiler/compiler/BUILD.bazel b/compiler/compiler/BUILD.bazel index ac0710c36..cca049428 100644 --- a/compiler/compiler/BUILD.bazel +++ b/compiler/compiler/BUILD.bazel @@ -233,3 +233,13 @@ native_binary( out = "local_valdi_compiler_native", visibility = ["//visibility:public"], ) + +filegroup( + name = "clientsql_generator_test_data", + srcs = [ + "Compiler/Sources/Processors/ClientSqlProcessor.swift", + "Compiler/Tests/CompilerTests/ClientSqlProcessorTests.swift", + ], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/compiler/compiler/Compiler/Sources/Config/ValdiProjectConfig.swift b/compiler/compiler/Compiler/Sources/Config/ValdiProjectConfig.swift index 48645c161..50ca2771f 100644 --- a/compiler/compiler/Compiler/Sources/Config/ValdiProjectConfig.swift +++ b/compiler/compiler/Compiler/Sources/Config/ValdiProjectConfig.swift @@ -68,6 +68,8 @@ struct ValdiProjectConfig { let minifyConfigURL: URL? let clientSqlURL: URL? + + let clientSqlValidatorURL: URL? let sqlExportPathTemplate: String? @@ -291,6 +293,16 @@ struct ValdiProjectConfig { .map { try $0.resolvingVariables(environment) } .flatMap { configDirectoryUrl.resolving(path: $0, isDirectory: false) } } + + var clientSqlValidatorURL: URL? + if let path = args.directClientSqlValidatorPath { + logger.info("Using direct clientsql SQLite validator: \(path)") + clientSqlValidatorURL = currentDirectoryUrl.resolving(path: path) + } else { + clientSqlValidatorURL = try config["clientsql_sqlite_validator_path"]?.string + .map { try $0.resolvingVariables(environment) } + .flatMap { configDirectoryUrl.resolving(path: $0, isDirectory: false) } + } let sqlExportPathTemplate = config["sql_export_path_template"]?.string @@ -357,6 +369,7 @@ struct ValdiProjectConfig { compilerToolboxURL: compilerToolboxURL, minifyConfigURL: minifyConfigURL, clientSqlURL: clientSqlURL, + clientSqlValidatorURL: clientSqlValidatorURL, sqlExportPathTemplate: sqlExportPathTemplate, androidDefaultClassPath: androidDefaultClassPath, cppDefaultClassPrefix: cppDefaultClassPrefix, diff --git a/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift b/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift index 11b223f6c..dc5f72ed7 100644 --- a/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift +++ b/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift @@ -23,6 +23,21 @@ final class ClientSqlProcessor: CompilationProcessor { private var bundleToSourceMap: Synchronized<[String: [RelativePath: String]]> = Synchronized(data: [:]) private let diskCache: DiskCache? + static func compilerArguments(sqlDirectory: String, + package: String, + moduleName: String, + outputDirectory: String, + sqliteValidatorPath: String, + typeMapping: [String]) -> [String] { + return ["--sqlite-validator", sqliteValidatorPath, + "-s", sqlDirectory, + "-p", package, + "-c", package, + "-m", moduleName, + "-o", outputDirectory, + "-l", "typescript"] + typeMapping + } + init(logger: ILogger, fileManager: ValdiFileManager, diskCacheProvider: DiskCacheProvider, @@ -35,9 +50,14 @@ final class ClientSqlProcessor: CompilationProcessor { self.rootBundle = rootBundle if let clientSqlCompilerPath = projectConfig.clientSqlURL?.path, diskCacheProvider.isEnabled() { - logger.debug("Resolving SQL version") - let output = try SyncProcessHandle.run(logger: logger, command: clientSqlCompilerPath, arguments: ["-version"]).trimmed - diskCache = diskCacheProvider.newCache(cacheName: "clientsql", outputExtension: "ts", metadata: ["version": output]) + logger.debug("Resolving ClientSQL generator artifact identity") + let sqliteValidatorPath = try projectConfig.ensureClientSqlValidator() + let output = try SyncProcessHandle.run( + logger: logger, + command: clientSqlCompilerPath, + arguments: ["--sqlite-validator", sqliteValidatorPath, "-version"] + ).trimmed + diskCache = diskCacheProvider.newCache(cacheName: "clientsql", outputExtension: "ts", metadata: ["artifact": output]) } else { diskCache = nil } @@ -132,6 +152,7 @@ final class ClientSqlProcessor: CompilationProcessor { // Run the external compiler let compilerPath = try projectConfig.ensureClientSqlCompiler() + let sqliteValidatorPath = try projectConfig.ensureClientSqlValidator() // Clear any stale files from prior runs so the post-compile enumerator // only picks up files emitted by this invocation. The directory path is @@ -143,12 +164,12 @@ final class ClientSqlProcessor: CompilationProcessor { } } - let args = ["-s", sqlDir, - "-p", pkg, - "-c", clazz, - "-m", clazz, - "-o", outputDirectory.path, - "-l", "typescript"] + typemapping + let args = Self.compilerArguments(sqlDirectory: sqlDir, + package: clazz, + moduleName: bundleInfo.name, + outputDirectory: outputDirectory.path, + sqliteValidatorPath: sqliteValidatorPath, + typeMapping: typemapping) let output = try SyncProcessHandle.run(logger: logger, command: compilerPath, arguments: args) logger.verbose("-- CLIENTSQL compiler:") @@ -209,6 +230,14 @@ extension ValdiProjectConfig { return path } + func ensureClientSqlValidator() throws -> String { + guard let path = clientSqlValidatorURL?.path else { + throw CompilerError("ClientSQL SQLite 3.16 validator path is not defined") + } + + return path + } + func globalMetadataURL(for platform: Platform) throws -> URL? { let output: ValdiOutputConfig? switch platform { diff --git a/compiler/compiler/Compiler/Sources/ValdiCompilerArguments.swift b/compiler/compiler/Compiler/Sources/ValdiCompilerArguments.swift index 573203dfd..af012f48d 100644 --- a/compiler/compiler/Compiler/Sources/ValdiCompilerArguments.swift +++ b/compiler/compiler/Compiler/Sources/ValdiCompilerArguments.swift @@ -193,6 +193,9 @@ struct ValdiCompilerArguments: ParsableCommand { @Option(help: "path to the clientsql app") var directClientSqlPath: String? + @Option(help: "path to the ClientSQL SQLite 3.16 validator") + var directClientSqlValidatorPath: String? + @Option(help: "path to the file with the list of input files") var explicitInputListFile: String? diff --git a/compiler/compiler/Compiler/Tests/CompilerTests/ClientSqlProcessorTests.swift b/compiler/compiler/Compiler/Tests/CompilerTests/ClientSqlProcessorTests.swift new file mode 100644 index 000000000..e143c05de --- /dev/null +++ b/compiler/compiler/Compiler/Tests/CompilerTests/ClientSqlProcessorTests.swift @@ -0,0 +1,27 @@ +import XCTest +@testable import Compiler + +final class ClientSqlProcessorTests: XCTestCase { + func testCompilerArgumentsKeepBundleIdentitySeparateFromDatabaseClass() { + XCTAssertEqual( + ClientSqlProcessor.compilerArguments( + sqlDirectory: "/project/MyBundle/sql", + package: "SharedDb", + moduleName: "MyBundle", + outputDirectory: "/generated/MyBundle/src/sqlgen", + sqliteValidatorPath: "/tools/sqlite_316_validator", + typeMapping: ["-tm", "sql_types.yaml"] + ), + [ + "--sqlite-validator", "/tools/sqlite_316_validator", + "-s", "/project/MyBundle/sql", + "-p", "SharedDb", + "-c", "SharedDb", + "-m", "MyBundle", + "-o", "/generated/MyBundle/src/sqlgen", + "-l", "typescript", + "-tm", "sql_types.yaml", + ] + ) + } +} diff --git a/fossa-deps.yml b/fossa-deps.yml index 0e647522b..bc323ffd5 100644 --- a/fossa-deps.yml +++ b/fossa-deps.yml @@ -67,6 +67,12 @@ remote-dependencies: - name: fmt version: 7.1.3 url: https://github.com/fmtlib/fmt/releases/download/7.1.3/fmt-7.1.3.zip +- name: sqlite + version: 3.53.4 + url: https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz +- name: sqlite-clientsql-validator + version: 3.16.0 + url: https://www.sqlite.org/2017/sqlite-amalgamation-3160000.zip - name: jsoncpp version: 1.8.0 url: https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.8.0.zip diff --git a/src/valdi_modules/src/valdi/client_sql/BUILD.bazel b/src/valdi_modules/src/valdi/client_sql/BUILD.bazel new file mode 100644 index 000000000..fef2b4f43 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/BUILD.bazel @@ -0,0 +1,86 @@ +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") +load("//bzl/valdi:valdi_module.bzl", "valdi_module") +load("//valdi:valdi.bzl", "valdi_test") + +cc_library( + name = "client_sql_native_impl", + srcs = ["native/ClientSQLNativeModuleFactory.cpp"], + hdrs = ["native/ClientSQLNativeModuleFactory.hpp"], + alwayslink = 1, + linkopts = select({ + "//bzl/conditions:ios": ["-lsqlite3"], + "//bzl/conditions:macos": ["-lsqlite3"], + "//conditions:default": [], + }), + local_defines = select({ + "//bzl/conditions:ios": [], + "//bzl/conditions:macos": [], + "//conditions:default": ["VALDI_CLIENTSQL_USE_BUNDLED_SQLITE"], + }), + visibility = ["//visibility:public"], + deps = [ + ":client_sql_cpp", + "//valdi:valdi_runtime", + ] + select({ + "//bzl/conditions:ios": [], + "//bzl/conditions:macos": [], + "//conditions:default": ["@sqlite//:sqlite"], + }), +) + +valdi_test( + name = "client_sql_native_tests", + srcs = ["native/ClientSQLNativeModuleFactory_tests.cpp"], + deps = [ + ":client_sql_native_impl", + "//valdi:test_utils", + "//valdi:valdi_runtime", + ], +) + +ts_project( + name = "client_sql_web", + srcs = glob([ + "web/**/*.ts", + "src/**/*.d.ts", + ]), + allow_js = True, + composite = True, + declaration = True, + transpiler = "tsc", + tsconfig = "web/tsconfig.json", +) + +valdi_module( + name = "client_sql", + srcs = glob([ + "src/**/*.ts", + "src/**/*.tsx", + "test/**/*.ts", + "test/**/*.tsx", + ]) + [ + "tsconfig.json", + ], + android_output_target = "release", + ios_module_name = "SCCClientSQL", + ios_output_target = "release", + native_deps = [":client_sql_native_impl"], + visibility = ["//visibility:public"], + web_deps = [":client_sql_web"], + web_register_native_module_id_overrides = { + "client_sql/web/ClientSQLNative.js": "client_sql/src/ClientSQLNative", + }, + deps = [ + "//src/valdi_modules/src/valdi/valdi_core", + ], +) + +filegroup( + name = "clientsql_generator_test_data", + srcs = [ + "BUILD.bazel", + "src/ClientSQLNative.d.ts", + ], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/src/valdi_modules/src/valdi/client_sql/README.md b/src/valdi_modules/src/valdi/client_sql/README.md new file mode 100644 index 000000000..939361907 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/README.md @@ -0,0 +1,33 @@ +# ClientSQL runtime integration boundaries + +ClientSQL's debugger adapter intentionally imports the generic +`valdi_core/src/debugging/DebuggerProvider` contract. That source is supplied by +the reviewed debugger-provider stack and is not duplicated in this optional +lane. Restack this branch on the provider branch before Valdi typecheck or Bazel +validation. + +The adapter returns the generic contract's `{ json: string }` result from one +helper. That helper's only input is ClientSQL's bounded, pre-serialized JSON +document (40 KiB maximum, below the generic provider's 48 KiB action-document +limit), and the generic provider validates the document without traversing a +provider-owned object graph. The generic provider owns the exact 128 KiB final +`{ handled, data }` response limit, transport, and request routing; ClientSQL +defines no fallback protocol. Every serialized array and object is capped at +100 items/properties to match the generic provider parser. Truncation is +deterministic and reports its omitted-value count and `collectionValues` or +`objectProperties` reason in the document metadata. + +Table actions return each row as a positional value array aligned with the +separately bounded `columns` metadata. Arbitrarily long valid SQL identifiers +therefore remain string values and never become JSON property names, avoiding +the generic provider's 1024-character property-name boundary without reducing +the core ClientSQL identifier contract. + +The adapter creates its provider owner with the actual ClientSQL module object +and the stable `client_sql/src/ClientSQLDebug` owner key. The generic provider +binds that owner to Valdi's module-loader hot-reload callback. Reload disposal is +therefore automatic even when `module.path` is absent, and same-key replacement +permanently retires the old owner before the new module registers the one active +`client-sql` provider. ClientSQL does not add a second manual disposal protocol; +closing the final live database only disposes the current registration so the +same loaded module can register again later. diff --git a/src/valdi_modules/src/valdi/client_sql/module.yaml b/src/valdi_modules/src/valdi/client_sql/module.yaml new file mode 100644 index 000000000..576220830 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/module.yaml @@ -0,0 +1,8 @@ +name: client_sql +ios: + module_name: SCCClientSQL + output: release +android: + output: release +dependencies: + - valdi_core diff --git a/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.cpp b/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.cpp new file mode 100644 index 000000000..dee359113 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.cpp @@ -0,0 +1,1670 @@ +#include "ClientSQLNativeModuleFactory.hpp" + +#include "valdi/runtime/Runtime.hpp" +#include "valdi_core/cpp/JavaScript/ModuleFactoryRegistry.hpp" +#include "valdi_core/cpp/Threading/DispatchQueue.hpp" +#include "valdi_core/cpp/Utils/DiskUtils.hpp" +#include "valdi_core/cpp/Utils/Exception.hpp" +#include "valdi_core/cpp/Utils/Format.hpp" +#include "valdi_core/cpp/Utils/StringCache.hpp" +#include "valdi_core/cpp/Utils/ValueArray.hpp" +#include "valdi_core/cpp/Utils/ValueMap.hpp" +#include "valdi_core/cpp/Utils/ValueTypedArray.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(VALDI_CLIENTSQL_USE_BUNDLED_SQLITE) +#include "sqlite3.h" +#else +#include +#endif + +namespace Valdi { +namespace { + +constexpr std::string_view kClientSQLDirectory = "ClientSQLNative"; +constexpr size_t kClientSQLReaderConnectionCount = 4; +constexpr sqlite3_int64 kMaxSafeJavaScriptInteger = 9007199254740991LL; +constexpr int kMinimumClientSQLSQLiteVersion = 3016000; +std::atomic gClientSQLLiveCoordinatorCount{0}; + +struct SQLiteStatementDeleter { + void operator()(sqlite3_stmt* statement) const { + sqlite3_finalize(statement); + } +}; + +using SQLiteStatement = std::unique_ptr; + +using SQLiteConnectionHandle = std::unique_ptr; + +const char* clientSQLSQLiteError(sqlite3* database) { + return database == nullptr ? "SQLite error" : sqlite3_errmsg(database); +} + +sqlite3_destructor_type sqliteTransient() { + return reinterpret_cast(-1); +} + +Result checkSQLiteStatus(sqlite3* database, int status, std::string_view operation) { + if (status == SQLITE_OK || status == SQLITE_DONE || status == SQLITE_ROW) { + return Void(); + } + + return Error(STRING_FORMAT("ClientSQLNative {} failed: {}", operation, clientSQLSQLiteError(database))); +} + +bool execRawSQL(sqlite3* database, std::string_view sql, Error* outError) { + char* rawError = nullptr; + const auto status = sqlite3_exec(database, std::string(sql).c_str(), nullptr, nullptr, &rawError); + if (status == SQLITE_OK) { + return true; + } + + if (outError != nullptr) { + std::string message = rawError != nullptr ? rawError : clientSQLSQLiteError(database); + *outError = Error(STRING_FORMAT("ClientSQLNative SQL failed: {}", message)); + } + sqlite3_free(rawError); + return false; +} + +Result prepareStatement(sqlite3* database, const StringBox& sql) { + if (sql.length() > static_cast(std::numeric_limits::max())) { + return Error("ClientSQLNative SQL text is too large"); + } + + sqlite3_stmt* statement = nullptr; + const auto status = sqlite3_prepare_v2( + database, sql.getCStr(), static_cast(sql.length()), &statement, nullptr); + if (status != SQLITE_OK) { + return Error(STRING_FORMAT("ClientSQLNative prepare failed: {}", clientSQLSQLiteError(database))); + } + + return SQLiteStatement(statement); +} + +Result readUserVersion(sqlite3* database) { + auto statementResult = prepareStatement(database, STRING_LITERAL("PRAGMA user_version")); + if (!statementResult) { + return statementResult.moveError(); + } + + auto statement = statementResult.moveValue(); + const auto status = sqlite3_step(statement.get()); + if (status != SQLITE_ROW) { + return Error(STRING_FORMAT("ClientSQLNative failed to read schema version: {}", clientSQLSQLiteError(database))); + } + + return sqlite3_column_int(statement.get(), 0); +} + +Result enableAndVerifyWAL(sqlite3* database) { + auto statementResult = prepareStatement(database, STRING_LITERAL("PRAGMA journal_mode = WAL")); + if (!statementResult) { + return statementResult.moveError(); + } + + auto statement = statementResult.moveValue(); + const auto status = sqlite3_step(statement.get()); + if (status != SQLITE_ROW) { + return Error(STRING_FORMAT( + "ClientSQLNative failed to enable WAL mode: {}", clientSQLSQLiteError(database))); + } + + const auto* mode = sqlite3_column_text(statement.get(), 0); + const auto modeLength = sqlite3_column_bytes(statement.get(), 0); + if (mode == nullptr || modeLength != 3 || sqlite3_strnicmp(reinterpret_cast(mode), "wal", 3) != 0) { + const auto actualMode = mode == nullptr + ? std::string("") + : std::string(reinterpret_cast(mode), static_cast(std::max(modeLength, 0))); + return Error(STRING_FORMAT( + "ClientSQLNative requires WAL journal mode but SQLite selected '{}'", actualMode)); + } + + return Void(); +} + +Error makeSQLExecutionError(sqlite3* database, std::string_view operation) { + return Error(STRING_FORMAT("ClientSQLNative {} failed: {}", operation, clientSQLSQLiteError(database))); +} + +Result configureSQLiteConnection(sqlite3* database, bool writer) { + if (sqlite3_libversion_number() < kMinimumClientSQLSQLiteVersion) { + return Error(STRING_FORMAT( + "ClientSQLNative requires SQLite 3.16.0 or newer; found {}", sqlite3_libversion())); + } + sqlite3_busy_timeout(database, 5000); + + Error sqlError; + if (writer) { + auto walResult = enableAndVerifyWAL(database); + if (!walResult) { + return walResult.moveError(); + } + } + if (!execRawSQL(database, "PRAGMA foreign_keys = ON", &sqlError)) { + return sqlError; + } + if (!execRawSQL(database, "PRAGMA synchronous = NORMAL", &sqlError)) { + return sqlError; + } + if (!writer && !execRawSQL(database, "PRAGMA query_only = ON", &sqlError)) { + return sqlError; + } + + return Void(); +} + +Result openSQLiteDatabase(const std::string& databasePath, + const StringBox& name, + int flags, + bool writer) { + sqlite3* rawDatabase = nullptr; + const auto status = sqlite3_open_v2(databasePath.c_str(), &rawDatabase, flags | SQLITE_OPEN_FULLMUTEX, nullptr); + SQLiteConnectionHandle database(rawDatabase, sqlite3_close); + if (status != SQLITE_OK) { + return Error( + STRING_FORMAT("ClientSQLNative failed to open database '{}': {}", name, clientSQLSQLiteError(rawDatabase))); + } + + auto configureResult = configureSQLiteConnection(database.get(), writer); + if (!configureResult) { + return configureResult.moveError(); + } + + return std::move(database); +} + +Result bindParameters(sqlite3* database, sqlite3_stmt* statement, const Value& parametersValue) { + const auto bindCount = sqlite3_bind_parameter_count(statement); + Ref parameters; + if (parametersValue.isNullOrUndefined()) { + parameters = ValueArray::make(0); + } else if (parametersValue.isArray()) { + parameters = parametersValue.getArrayRef(); + } else { + return Error("ClientSQLNative parameters must be an array"); + } + + if (parameters->size() != static_cast(bindCount)) { + return Error( + STRING_FORMAT("ClientSQLNative expected {} SQL parameters but received {}", bindCount, parameters->size())); + } + + for (int parameterIndex = 0; parameterIndex < bindCount; ++parameterIndex) { + const auto& parameter = (*parameters)[static_cast(parameterIndex)]; + const auto sqliteIndex = parameterIndex + 1; + int status = SQLITE_OK; + + switch (parameter.getType()) { + case ValueType::Null: + status = sqlite3_bind_null(statement, sqliteIndex); + break; + case ValueType::Bool: + status = sqlite3_bind_int(statement, sqliteIndex, parameter.toBool() ? 1 : 0); + break; + case ValueType::Int: + status = sqlite3_bind_int(statement, sqliteIndex, parameter.toInt()); + break; + case ValueType::Long: + status = sqlite3_bind_int64(statement, sqliteIndex, static_cast(parameter.toLong())); + break; + case ValueType::Double: { + const auto value = parameter.toDouble(); + if (!std::isfinite(value)) { + return Error("ClientSQLNative double parameters must be finite"); + } + status = sqlite3_bind_double(statement, sqliteIndex, value); + break; + } + case ValueType::InternedString: + case ValueType::StaticString: { + auto string = parameter.toStringBox(); + if (string.length() > static_cast(std::numeric_limits::max())) { + return Error("ClientSQLNative string parameter is too large"); + } + status = sqlite3_bind_text( + statement, sqliteIndex, string.getCStr(), static_cast(string.length()), sqliteTransient()); + break; + } + case ValueType::TypedArray: { + const auto& buffer = parameter.getTypedArray()->getBuffer(); + if (buffer.size() > static_cast(std::numeric_limits::max())) { + return Error("ClientSQLNative blob parameter is too large"); + } + static constexpr Byte emptyBlobSentinel = 0; + const auto* blobData = buffer.size() == 0 ? &emptyBlobSentinel : buffer.data(); + status = sqlite3_bind_blob( + statement, + sqliteIndex, + blobData, + static_cast(buffer.size()), + sqliteTransient()); + break; + } + default: + return Error(STRING_FORMAT( + "ClientSQLNative unsupported parameter type '{}'", valueTypeToString(parameter.getType()))); + } + + auto bindResult = checkSQLiteStatus(database, status, "bind"); + if (!bindResult) { + return bindResult; + } + } + + return Void(); +} + +Result sqliteColumnToValue(sqlite3_stmt* statement, int columnIndex) { + switch (sqlite3_column_type(statement, columnIndex)) { + case SQLITE_NULL: + return Value(); + case SQLITE_INTEGER: { + const auto value = sqlite3_column_int64(statement, columnIndex); + if (value >= std::numeric_limits::min() && value <= std::numeric_limits::max()) { + return Value(static_cast(value)); + } + if (value < -kMaxSafeJavaScriptInteger || value > kMaxSafeJavaScriptInteger) { + return Error(STRING_FORMAT( + "ClientSQLNative integer value {} exceeds JavaScript's exact integer range; store 64-bit identifiers as TEXT", + value)); + } + return Value(static_cast(value)); + } + case SQLITE_FLOAT: + return Value(sqlite3_column_double(statement, columnIndex)); + case SQLITE_TEXT: { + const auto* text = sqlite3_column_text(statement, columnIndex); + const auto byteCount = sqlite3_column_bytes(statement, columnIndex); + if (byteCount < 0) { + return Error(STRING_FORMAT( + "ClientSQLNative SQLite TEXT column {} returned an invalid length", columnIndex)); + } + if (byteCount == 0) { + return Value(StringBox::emptyString()); + } + if (text == nullptr) { + return Error(STRING_FORMAT( + "ClientSQLNative SQLite TEXT column {} returned null data for {} bytes", columnIndex, byteCount)); + } + return Value(StringCache::getGlobal().makeString( + std::string_view(reinterpret_cast(text), static_cast(byteCount)))); + } + case SQLITE_BLOB: { + const auto* data = sqlite3_column_blob(statement, columnIndex); + const auto byteCount = sqlite3_column_bytes(statement, columnIndex); + if (byteCount < 0) { + return Error(STRING_FORMAT( + "ClientSQLNative SQLite BLOB column {} returned an invalid length", columnIndex)); + } + if (byteCount > 0 && data == nullptr) { + return Error(STRING_FORMAT( + "ClientSQLNative SQLite BLOB column {} returned null data for {} bytes", columnIndex, byteCount)); + } + auto bytes = makeShared(); + if (byteCount > 0) { + bytes->assignData(reinterpret_cast(data), static_cast(byteCount)); + } + return Value(makeShared(TypedArrayType::ArrayBuffer, bytes)); + } + default: + return Value::undefined(); + } +} + +Result sqliteRowToValue(sqlite3_stmt* statement) { + auto row = makeShared(); + const auto columnCount = sqlite3_column_count(statement); + for (int columnIndex = 0; columnIndex < columnCount; ++columnIndex) { + auto columnName = + StringCache::getGlobal().makeString(std::string_view(sqlite3_column_name(statement, columnIndex))); + auto columnValue = sqliteColumnToValue(statement, columnIndex); + if (!columnValue) { + return columnValue.moveError(); + } + (*row)[columnName] = columnValue.moveValue(); + } + return Value(row); +} + +struct ClientSQLRequest { + StringBox sql; + Value parameters; +}; + +struct ClientSQLDatabasePath { + std::string databasePath; + Path databaseRoot; +}; + +struct ClientSQLOpenRequest { + StringBox name; + ClientSQLDatabasePath databasePath; + int32_t schemaVersion; + Ref createStatements; + Ref migrations; +}; + +void appendSchemaFingerprintString(std::string& fingerprint, const StringBox& value) { + const auto view = value.toStringView(); + fingerprint.append(std::to_string(view.size())); + fingerprint.push_back(':'); + fingerprint.append(view.data(), view.size()); +} + +std::string schemaFingerprint(const ClientSQLOpenRequest& request) { + std::string fingerprint; + fingerprint.append("schema-version:"); + fingerprint.append(std::to_string(request.schemaVersion)); + fingerprint.append(";create-count:"); + fingerprint.append(std::to_string(request.createStatements->size())); + fingerprint.push_back(';'); + for (const auto& statement : *request.createStatements) { + appendSchemaFingerprintString(fingerprint, statement.toStringBox()); + } + fingerprint.append("migration-count:"); + fingerprint.append(std::to_string(request.migrations->size())); + fingerprint.push_back(';'); + for (const auto& migration : *request.migrations) { + fingerprint.append("migration-version:"); + fingerprint.append(std::to_string(migration.getMapValue("version").toInt())); + fingerprint.push_back(';'); + auto statements = migration.getMapValue("statements").getArrayRef(); + const auto statementCount = statements == nullptr ? 0 : statements->size(); + fingerprint.append("statement-count:"); + fingerprint.append(std::to_string(statementCount)); + fingerprint.push_back(';'); + if (statements != nullptr) { + for (const auto& statement : *statements) { + appendSchemaFingerprintString(fingerprint, statement.toStringBox()); + } + } + } + return fingerprint; +} + +class ClientSQLQueueConnection : public SimpleRefCountable { +public: + explicit ClientSQLQueueConnection(Ref queue) + : database(nullptr, sqlite3_close), queue(std::move(queue)) {} + ~ClientSQLQueueConnection() override = default; + + Result requireDatabase() { + if (!openError.isEmpty()) { + return openError; + } + if (database == nullptr) { + return Error("ClientSQLNative database is not open"); + } + return database.get(); + } + + SQLiteConnectionHandle database; + Ref queue; + Error openError; +}; + +Result applySchema(sqlite3* database, + int32_t currentVersion, + int32_t targetVersion, + const Ref& createStatements, + const Ref& migrations); +Result openWriterDatabase(const Ref& connection, const ClientSQLOpenRequest& request); +Result openReaderDatabases(const std::vector>& readerConnections, + const ClientSQLOpenRequest& request); + +Result executeStatement(sqlite3* database, const ClientSQLRequest& request) { + auto statement = prepareStatement(database, request.sql); + if (!statement) { + return statement.moveError(); + } + + auto bindResult = bindParameters(database, statement.value().get(), request.parameters); + if (!bindResult) { + return bindResult.moveError(); + } + + while (true) { + const auto status = sqlite3_step(statement.value().get()); + if (status == SQLITE_DONE) { + return Void(); + } + if (status != SQLITE_ROW) { + return makeSQLExecutionError(database, "execute"); + } + } +} + +Result queryStatement(sqlite3* database, const ClientSQLRequest& request) { + auto statement = prepareStatement(database, request.sql); + if (!statement) { + return statement.moveError(); + } + + auto bindResult = bindParameters(database, statement.value().get(), request.parameters); + if (!bindResult) { + return bindResult.moveError(); + } + + std::vector rows; + while (true) { + const auto status = sqlite3_step(statement.value().get()); + if (status == SQLITE_DONE) { + return Value(ValueArray::make(std::move(rows))); + } + if (status != SQLITE_ROW) { + return makeSQLExecutionError(database, "query"); + } + auto row = sqliteRowToValue(statement.value().get()); + if (!row) { + return row.moveError(); + } + rows.emplace_back(row.moveValue()); + } +} + +using ClientSQLValueCallback = + snap::valdi_modules::client_sql::ClientSQLNativeConnectionProxy::ExecuteCallbackFn; +using ClientSQLArrayCallback = + snap::valdi_modules::client_sql::ClientSQLNativeConnectionProxy::QueryCallbackFn; + +void notifyClientSQLCallbackSuccess( + const ClientSQLValueCallback& callback, + Value value) { + callback(std::move(value), std::nullopt); +} + +void notifyClientSQLCallbackError(const ClientSQLValueCallback& callback, Error error) { + callback(Value::undefined(), error.getMessage()); +} + +void notifyClientSQLCallbackSuccess( + const ClientSQLArrayCallback& callback, + Value value) { + auto array = value.getArrayRef(); + if (array == nullptr) { + callback(std::nullopt, std::nullopt); + return; + } + callback(std::vector(array->begin(), array->end()), std::nullopt); +} + +void notifyClientSQLCallbackError(const ClientSQLArrayCallback& callback, Error error) { + callback(std::nullopt, error.getMessage()); +} + +Value makeClientSQLParameters(std::optional> parameters) { + if (!parameters.has_value()) { + return Value::undefined(); + } + return Value(ValueArray::make(std::move(parameters.value()))); +} + +class ClientSQLConnection; +class ClientSQLTransaction; + +class ClientSQLDatabaseCoordinator : public SharedPtrRefCountable { +public: + explicit ClientSQLDatabaseCoordinator(const Ref& fallbackQueue); + ~ClientSQLDatabaseCoordinator() override; + + bool hasWriterQueue() const; + Ref openHandle(ClientSQLOpenRequest request); + void execute(const Ref& connection, ClientSQLValueCallback callback, ClientSQLRequest request); + void query(const Ref& connection, ClientSQLArrayCallback callback, ClientSQLRequest request); + void queryOnWriter( + const Ref& connection, + ClientSQLArrayCallback callback, + ClientSQLRequest request); + void transaction( + const Ref& connection, + snap::valdi_modules::client_sql::ClientSQLNativeConnectionProxy::TransactionBodyFn body, + ClientSQLValueCallback callback); + void debugInfo(const Ref& connection, ClientSQLValueCallback callback); + void executeInTransaction(const Ref& connection, + uint64_t transactionId, + ClientSQLValueCallback callback, + ClientSQLRequest request); + void queryInTransaction(const Ref& connection, + uint64_t transactionId, + ClientSQLArrayCallback callback, + ClientSQLRequest request); + void finishTransaction( + uint64_t transactionId, + ClientSQLValueCallback callback, + Value value, + std::optional error); + void closeHandle(const Ref& connection, ClientSQLValueCallback callback); + bool isTransactionOwner(const ClientSQLConnection* connection) const; + +private: + Result ensureOpen(const ClientSQLOpenRequest& request); + Ref nextReaderConnection(); + void closeAllConnectionsOnWriterThread(); + void enqueueWriterWork(DispatchFunction work, bool allowDuringTransaction); + void runWriterWorkOnWriterThread(DispatchFunction work, bool allowDuringTransaction); + void drainDeferredWriterWorkOnWriterThread(); + void finishTransactionOnWriterThread( + uint64_t transactionId, + ClientSQLValueCallback callback, + Value value, + std::optional error); + bool hasActiveTransaction(uint64_t transactionId) const; + + Ref _writerConnection; + std::vector> _readerConnections; + std::atomic _activeHandles{0}; + std::atomic _nextReader{0}; + std::atomic_bool _readerConnectionsReady{false}; + uint64_t _activeTransactionId = 0; + std::atomic _activeTransactionConnection{nullptr}; + uint64_t _nextTransactionId = 1; + std::deque _deferredWriterWork; + std::optional _schemaFingerprint; +}; + +class ClientSQLConnection final + : public snap::valdi_modules::client_sql::ClientSQLNativeConnectionProxy { +public: + explicit ClientSQLConnection(Ref coordinator) + : _coordinator(std::move(coordinator)) {} + ~ClientSQLConnection() override = default; + + bool isClosed() const { + return _closed.load(); + } + + void markOpenSuccess() { + std::lock_guard lock(_openMutex); + _openError = Error(); + _openComplete.store(true); + } + + void markOpenError(Error error) { + std::lock_guard lock(_openMutex); + _openError = std::move(error); + _openComplete.store(true); + } + + Result requireOpen() const { + if (!_openComplete.load()) { + return Error("ClientSQLNative database is not open"); + } + + std::lock_guard lock(_openMutex); + if (!_openError.isEmpty()) { + return _openError; + } + return Void(); + } + + void execute( + StringBox sql, + std::optional> parameters, + ExecuteCallbackFn callback) final { + if (isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto coordinator = coordinatorRef(); + if (coordinator == nullptr) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + if (coordinator->isTransactionOwner(this)) { + notifyClientSQLCallbackError( + callback, + Error("ClientSQLNative parent connection execute is not allowed inside its transaction body")); + return; + } + coordinator->execute( + strongSmallRef(this), + std::move(callback), + ClientSQLRequest{ + .sql = std::move(sql), + .parameters = makeClientSQLParameters(std::move(parameters))}); + } + + void query( + StringBox sql, + std::optional> parameters, + QueryCallbackFn callback) final { + if (isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto coordinator = coordinatorRef(); + if (coordinator == nullptr) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + if (coordinator->isTransactionOwner(this)) { + notifyClientSQLCallbackError( + callback, + Error("ClientSQLNative parent connection query is not allowed inside its transaction body")); + return; + } + coordinator->query( + strongSmallRef(this), + std::move(callback), + ClientSQLRequest{ + .sql = std::move(sql), + .parameters = makeClientSQLParameters(std::move(parameters))}); + } + + void queryOnWriter( + StringBox sql, + std::optional> parameters, + QueryOnWriterCallbackFn callback) final { + if (isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto coordinator = coordinatorRef(); + if (coordinator == nullptr) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + if (coordinator->isTransactionOwner(this)) { + notifyClientSQLCallbackError( + callback, + Error("ClientSQLNative parent connection query is not allowed inside its transaction body")); + return; + } + coordinator->queryOnWriter( + strongSmallRef(this), + std::move(callback), + ClientSQLRequest{ + .sql = std::move(sql), + .parameters = makeClientSQLParameters(std::move(parameters))}); + } + + void transaction(TransactionBodyFn body, TransactionCallbackFn callback) final { + if (isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto coordinator = coordinatorRef(); + if (coordinator == nullptr) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + if (coordinator->isTransactionOwner(this)) { + notifyClientSQLCallbackError( + callback, + Error("ClientSQLNative nested parent connection transaction is not allowed inside a transaction body")); + return; + } + coordinator->transaction(strongSmallRef(this), std::move(body), std::move(callback)); + } + + void debugInfo(DebugInfoCallbackFn callback) final { + auto coordinator = coordinatorRef(); + if (coordinator == nullptr) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + coordinator->debugInfo(strongSmallRef(this), std::move(callback)); + } + + void close(CloseCallbackFn callback) final { + if (_closed.load()) { + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + return; + } + + auto coordinator = coordinatorRef(); + if (coordinator == nullptr) { + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + return; + } + if (coordinator->isTransactionOwner(this)) { + notifyClientSQLCallbackError( + callback, + Error("ClientSQLNative parent connection close is not allowed inside its transaction body")); + return; + } + if (_closed.exchange(true)) { + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + return; + } + coordinator = releaseCoordinator(); + if (coordinator == nullptr) { + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + return; + } + coordinator->closeHandle(strongSmallRef(this), std::move(callback)); + } + +private: + Ref coordinatorRef() const { + std::lock_guard lock(_coordinatorMutex); + return _coordinator; + } + + Ref releaseCoordinator() { + std::lock_guard lock(_coordinatorMutex); + auto coordinator = std::move(_coordinator); + _coordinator = nullptr; + return coordinator; + } + + mutable std::mutex _coordinatorMutex; + Ref _coordinator; + mutable std::mutex _openMutex; + Error _openError; + std::atomic_bool _openComplete{false}; + std::atomic_bool _closed{false}; +}; + +class ClientSQLTransaction final + : public snap::valdi_modules::client_sql::ClientSQLNativeTransactionProxy { +public: + ClientSQLTransaction(Ref coordinator, + Ref connection, + uint64_t transactionId) + : _coordinator(std::move(coordinator)), + _connection(std::move(connection)), + _transactionId(transactionId) {} + ~ClientSQLTransaction() override = default; + + void execute( + StringBox sql, + std::optional> parameters, + ExecuteCallbackFn callback) final { + _coordinator->executeInTransaction( + _connection, + _transactionId, + std::move(callback), + ClientSQLRequest{ + .sql = std::move(sql), + .parameters = makeClientSQLParameters(std::move(parameters))}); + } + + void query( + StringBox sql, + std::optional> parameters, + QueryCallbackFn callback) final { + _coordinator->queryInTransaction( + _connection, + _transactionId, + std::move(callback), + ClientSQLRequest{ + .sql = std::move(sql), + .parameters = makeClientSQLParameters(std::move(parameters))}); + } + +private: + Ref _coordinator; + Ref _connection; + uint64_t _transactionId; +}; + +ClientSQLDatabaseCoordinator::ClientSQLDatabaseCoordinator(const Ref& fallbackQueue) { + gClientSQLLiveCoordinatorCount.fetch_add(1); + if (fallbackQueue != nullptr) { + _writerConnection = makeShared(fallbackQueue); + } + + _readerConnections.reserve(kClientSQLReaderConnectionCount); + for (size_t index = 0; index < kClientSQLReaderConnectionCount; ++index) { + if (fallbackQueue != nullptr) { + _readerConnections.emplace_back(makeShared(fallbackQueue)); + } + } +} + +ClientSQLDatabaseCoordinator::~ClientSQLDatabaseCoordinator() { + gClientSQLLiveCoordinatorCount.fetch_sub(1); +} + +bool ClientSQLDatabaseCoordinator::hasWriterQueue() const { + return _writerConnection != nullptr && _writerConnection->queue != nullptr; +} + +Ref ClientSQLDatabaseCoordinator::openHandle(ClientSQLOpenRequest request) { + _activeHandles.fetch_add(1); + auto connection = makeShared(strongSmallRef(this)); + enqueueWriterWork([self = strongSmallRef(this), connection, request = std::move(request)]() { + if (connection->isClosed()) { + return; + } + + auto openResult = self->ensureOpen(request); + if (!openResult) { + connection->markOpenError(openResult.moveError()); + return; + } + connection->markOpenSuccess(); + }, false); + return connection; +} + +void ClientSQLDatabaseCoordinator::enqueueWriterWork(DispatchFunction work, bool allowDuringTransaction) { + _writerConnection->queue->async( + [self = strongSmallRef(this), work = std::move(work), allowDuringTransaction]() mutable { + self->runWriterWorkOnWriterThread(std::move(work), allowDuringTransaction); + }); +} + +void ClientSQLDatabaseCoordinator::runWriterWorkOnWriterThread(DispatchFunction work, bool allowDuringTransaction) { + if (!allowDuringTransaction && _activeTransactionId != 0) { + _deferredWriterWork.emplace_back(std::move(work)); + return; + } + + work(); +} + +void ClientSQLDatabaseCoordinator::drainDeferredWriterWorkOnWriterThread() { + while (_activeTransactionId == 0 && !_deferredWriterWork.empty()) { + auto work = std::move(_deferredWriterWork.front()); + _deferredWriterWork.pop_front(); + work(); + } +} + +Result ClientSQLDatabaseCoordinator::ensureOpen(const ClientSQLOpenRequest& request) { + const auto requestFingerprint = schemaFingerprint(request); + if (_schemaFingerprint.has_value() && _schemaFingerprint.value() != requestFingerprint) { + return Error(STRING_FORMAT( + "ClientSQLNative database '{}' is already open with a different schema fingerprint", + request.name)); + } + + if (_writerConnection->database == nullptr) { + auto writerOpenResult = openWriterDatabase(_writerConnection, request); + if (!writerOpenResult) { + return writerOpenResult.moveError(); + } + _writerConnection->openError = Error(); + _readerConnectionsReady.store(false); + } else { + auto currentVersion = readUserVersion(_writerConnection->database.get()); + if (!currentVersion) { + return currentVersion.moveError(); + } + + auto schemaResult = + applySchema(_writerConnection->database.get(), + currentVersion.value(), + request.schemaVersion, + request.createStatements, + request.migrations); + if (!schemaResult) { + return schemaResult.moveError(); + } + } + + _schemaFingerprint = requestFingerprint; + + if (!_readerConnections.empty() && !_readerConnectionsReady.load()) { + auto readerOpenResult = openReaderDatabases(_readerConnections, request); + if (readerOpenResult) { + _readerConnectionsReady.store(true); + } + } + + return Void(); +} + +void ClientSQLDatabaseCoordinator::execute(const Ref& connection, + ClientSQLValueCallback callback, + ClientSQLRequest request) { + enqueueWriterWork( + [self = strongSmallRef(this), connection, callback = std::move(callback), request = std::move(request)]() { + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto openResult = connection->requireOpen(); + if (!openResult) { + notifyClientSQLCallbackError(callback, openResult.moveError()); + return; + } + + auto database = self->_writerConnection->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + auto result = executeStatement(database.value(), request); + if (!result) { + notifyClientSQLCallbackError(callback, result.moveError()); + return; + } + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + }, false); +} + +void ClientSQLDatabaseCoordinator::query(const Ref& connection, + ClientSQLArrayCallback callback, + ClientSQLRequest request) { + _writerConnection->queue->async( + [self = strongSmallRef(this), connection, callback = std::move(callback), request = std::move(request)]() { + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto openResult = connection->requireOpen(); + if (!openResult) { + notifyClientSQLCallbackError(callback, openResult.moveError()); + return; + } + + auto reader = self->nextReaderConnection(); + if (reader.get() == self->_writerConnection.get()) { + auto database = reader->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + auto result = queryStatement(database.value(), request); + if (!result) { + notifyClientSQLCallbackError(callback, result.moveError()); + return; + } + notifyClientSQLCallbackSuccess(callback, result.moveValue()); + return; + } + + reader->queue->async([connection, callback, request = std::move(request), reader]() { + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto database = reader->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + auto result = queryStatement(database.value(), request); + if (!result) { + notifyClientSQLCallbackError(callback, result.moveError()); + return; + } + notifyClientSQLCallbackSuccess(callback, result.moveValue()); + }); + }); +} + +void ClientSQLDatabaseCoordinator::queryOnWriter(const Ref& connection, + ClientSQLArrayCallback callback, + ClientSQLRequest request) { + enqueueWriterWork( + [self = strongSmallRef(this), connection, callback = std::move(callback), request = std::move(request)]() { + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto openResult = connection->requireOpen(); + if (!openResult) { + notifyClientSQLCallbackError(callback, openResult.moveError()); + return; + } + + auto database = self->_writerConnection->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + auto result = queryStatement(database.value(), request); + if (!result) { + notifyClientSQLCallbackError(callback, result.moveError()); + return; + } + notifyClientSQLCallbackSuccess(callback, result.moveValue()); + }, false); +} + +void ClientSQLDatabaseCoordinator::transaction( + const Ref& connection, + snap::valdi_modules::client_sql::ClientSQLNativeConnectionProxy::TransactionBodyFn body, + ClientSQLValueCallback callback) { + enqueueWriterWork([self = strongSmallRef(this), connection, body = std::move(body), callback = std::move(callback)]() { + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto openResult = connection->requireOpen(); + if (!openResult) { + notifyClientSQLCallbackError(callback, openResult.moveError()); + return; + } + + auto database = self->_writerConnection->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + Error sqlError; + if (!execRawSQL(database.value(), "BEGIN TRANSACTION", &sqlError)) { + notifyClientSQLCallbackError(callback, sqlError); + return; + } + + const auto transactionId = self->_nextTransactionId++; + self->_activeTransactionId = transactionId; + self->_activeTransactionConnection.store(connection.get()); + auto transaction = makeShared(self, connection, transactionId); + auto completed = std::make_shared(false); + snap::valdi_modules::client_sql::ClientSQLNativeConnectionProxy::TransactionBodyCallbackFn doneCallback = + [self, transactionId, callback, completed](Value value, std::optional error) { + if (completed->exchange(true)) { + return; + } + + self->finishTransaction( + transactionId, + callback, + std::move(value), + std::move(error)); + }; + + try { + body(std::move(transaction), std::move(doneCallback)); + } catch (const Exception& error) { + if (completed->exchange(true)) { + return; + } + self->finishTransactionOnWriterThread( + transactionId, + callback, + Value::undefined(), + error.getMessage()); + } + }, false); +} + +void ClientSQLDatabaseCoordinator::debugInfo( + const Ref& connection, + ClientSQLValueCallback callback) { + enqueueWriterWork( + [self = strongSmallRef(this), connection, callback = std::move(callback)]() { + auto info = makeShared(); + (*info)[STRING_LITERAL("activeHandles")] = + Value(static_cast(self->_activeHandles.load())); + (*info)[STRING_LITERAL("activeTransaction")] = Value(self->_activeTransactionId != 0); + (*info)[STRING_LITERAL("deferredWriterWork")] = + Value(static_cast(self->_deferredWriterWork.size())); + (*info)[STRING_LITERAL("readerConnections")] = + Value(static_cast(self->_readerConnections.size())); + (*info)[STRING_LITERAL("readerConnectionsReady")] = Value(self->_readerConnectionsReady.load()); + (*info)[STRING_LITERAL("writerOpen")] = Value(self->_writerConnection->database != nullptr); + (*info)[STRING_LITERAL("connectionClosed")] = Value(connection->isClosed()); + (*info)[STRING_LITERAL("liveCoordinators")] = + Value(static_cast(gClientSQLLiveCoordinatorCount.load())); + (*info)[STRING_LITERAL("sqliteVersionNumber")] = Value(sqlite3_libversion_number()); + notifyClientSQLCallbackSuccess(callback, Value(info)); + }, + true); +} + +bool ClientSQLDatabaseCoordinator::hasActiveTransaction(uint64_t transactionId) const { + return _activeTransactionId == transactionId; +} + +bool ClientSQLDatabaseCoordinator::isTransactionOwner(const ClientSQLConnection* connection) const { + return connection != nullptr && _activeTransactionConnection.load() == connection; +} + +void ClientSQLDatabaseCoordinator::executeInTransaction(const Ref& connection, + uint64_t transactionId, + ClientSQLValueCallback callback, + ClientSQLRequest request) { + enqueueWriterWork( + [self = strongSmallRef(this), + connection, + transactionId, + callback = std::move(callback), + request = std::move(request)]() { + if (!self->hasActiveTransaction(transactionId)) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative transaction is no longer active")); + return; + } + + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto openResult = connection->requireOpen(); + if (!openResult) { + notifyClientSQLCallbackError(callback, openResult.moveError()); + return; + } + + auto database = self->_writerConnection->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + auto result = executeStatement(database.value(), request); + if (!result) { + notifyClientSQLCallbackError(callback, result.moveError()); + return; + } + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + }, + true); +} + +void ClientSQLDatabaseCoordinator::queryInTransaction(const Ref& connection, + uint64_t transactionId, + ClientSQLArrayCallback callback, + ClientSQLRequest request) { + enqueueWriterWork( + [self = strongSmallRef(this), + connection, + transactionId, + callback = std::move(callback), + request = std::move(request)]() { + if (!self->hasActiveTransaction(transactionId)) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative transaction is no longer active")); + return; + } + + if (connection->isClosed()) { + notifyClientSQLCallbackError(callback, Error("ClientSQLNative connection is closed")); + return; + } + + auto openResult = connection->requireOpen(); + if (!openResult) { + notifyClientSQLCallbackError(callback, openResult.moveError()); + return; + } + + auto database = self->_writerConnection->requireDatabase(); + if (!database) { + notifyClientSQLCallbackError(callback, database.moveError()); + return; + } + + auto result = queryStatement(database.value(), request); + if (!result) { + notifyClientSQLCallbackError(callback, result.moveError()); + return; + } + notifyClientSQLCallbackSuccess(callback, result.moveValue()); + }, + true); +} + +void ClientSQLDatabaseCoordinator::finishTransaction(uint64_t transactionId, + ClientSQLValueCallback callback, + Value value, + std::optional error) { + _writerConnection->queue->async( + [self = strongSmallRef(this), + transactionId, + callback = std::move(callback), + value = std::move(value), + error = std::move(error)]() mutable { + self->finishTransactionOnWriterThread(transactionId, callback, std::move(value), std::move(error)); + }); +} + +void ClientSQLDatabaseCoordinator::finishTransactionOnWriterThread(uint64_t transactionId, + ClientSQLValueCallback callback, + Value value, + std::optional error) { + if (!hasActiveTransaction(transactionId)) { + return; + } + + auto database = _writerConnection->requireDatabase(); + if (!database) { + _activeTransactionId = 0; + _activeTransactionConnection.store(nullptr); + notifyClientSQLCallbackError(callback, database.moveError()); + drainDeferredWriterWorkOnWriterThread(); + return; + } + + Error sqlError; + if (!error.has_value()) { + if (execRawSQL(database.value(), "COMMIT", &sqlError)) { + _activeTransactionId = 0; + _activeTransactionConnection.store(nullptr); + notifyClientSQLCallbackSuccess(callback, std::move(value)); + drainDeferredWriterWorkOnWriterThread(); + return; + } + + Error rollbackError; + execRawSQL(database.value(), "ROLLBACK", &rollbackError); + _activeTransactionId = 0; + _activeTransactionConnection.store(nullptr); + notifyClientSQLCallbackError(callback, sqlError); + drainDeferredWriterWorkOnWriterThread(); + return; + } + + if (!execRawSQL(database.value(), "ROLLBACK", &sqlError)) { + _activeTransactionId = 0; + _activeTransactionConnection.store(nullptr); + notifyClientSQLCallbackError(callback, sqlError); + drainDeferredWriterWorkOnWriterThread(); + return; + } + + _activeTransactionId = 0; + _activeTransactionConnection.store(nullptr); + notifyClientSQLCallbackError(callback, Error(std::move(error.value()))); + drainDeferredWriterWorkOnWriterThread(); +} + +void ClientSQLDatabaseCoordinator::closeHandle(const Ref& connection, + ClientSQLValueCallback callback) { + enqueueWriterWork([self = strongSmallRef(this), connection, callback = std::move(callback)]() { + (void)connection.get(); + if (self->_activeHandles.fetch_sub(1) == 1) { + self->closeAllConnectionsOnWriterThread(); + } + notifyClientSQLCallbackSuccess(callback, Value::undefined()); + }, false); +} + +Ref ClientSQLDatabaseCoordinator::nextReaderConnection() { + if (_readerConnections.empty() || !_readerConnectionsReady.load()) { + return _writerConnection; + } + const auto index = _nextReader.fetch_add(1) % _readerConnections.size(); + return _readerConnections[index]; +} + +void ClientSQLDatabaseCoordinator::closeAllConnectionsOnWriterThread() { + _writerConnection->database.reset(); + _writerConnection->openError = Error(); + _readerConnectionsReady.store(false); + for (const auto& reader : _readerConnections) { + auto resetReader = [reader]() { + reader->database.reset(); + reader->openError = Error(); + }; + if (reader->queue.get() == _writerConnection->queue.get()) { + resetReader(); + } else { + reader->queue->sync(resetReader); + } + } +} + +std::vector>> sortedMigrations(const Ref& migrations) { + std::vector>> out; + for (const auto& migrationValue : *migrations) { + auto version = migrationValue.getMapValue("version").toInt(); + auto statements = migrationValue.getMapValue("statements").getArrayRef(); + if (statements != nullptr) { + out.emplace_back(version, statements); + } + } + + std::sort(out.begin(), out.end(), [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); + return out; +} + +Result applySchema(sqlite3* database, + int32_t currentVersion, + int32_t targetVersion, + const Ref& createStatements, + const Ref& migrations) { + if (currentVersion > targetVersion) { + return Error(STRING_FORMAT( + "ClientSQLNative database schema version {} is newer than requested version {}", + currentVersion, + targetVersion)); + } + if (currentVersion == targetVersion) { + return Void(); + } + + Error sqlError; + if (!execRawSQL(database, "BEGIN TRANSACTION", &sqlError)) { + return sqlError; + } + + auto rollback = [&database]() { + Error ignoredError; + execRawSQL(database, "ROLLBACK", &ignoredError); + }; + + if (currentVersion == 0) { + for (const auto& statement : *createStatements) { + if (!execRawSQL(database, statement.toStringBox().toStringView(), &sqlError)) { + rollback(); + return sqlError; + } + } + } else { + auto expectedVersion = currentVersion + 1; + for (const auto& migration : sortedMigrations(migrations)) { + if (migration.first <= currentVersion || migration.first > targetVersion) { + continue; + } + if (migration.first != expectedVersion) { + rollback(); + return Error(STRING_FORMAT( + "ClientSQLNative missing migration for schema version {}", expectedVersion)); + } + for (const auto& statement : *migration.second) { + if (!execRawSQL(database, statement.toStringBox().toStringView(), &sqlError)) { + rollback(); + return sqlError; + } + } + expectedVersion++; + } + if (expectedVersion <= targetVersion) { + rollback(); + return Error(STRING_FORMAT( + "ClientSQLNative missing migration for schema version {}", expectedVersion)); + } + } + + if (!execRawSQL(database, STRING_FORMAT("PRAGMA user_version = {}", targetVersion).toStringView(), &sqlError)) { + rollback(); + return sqlError; + } + if (!execRawSQL(database, "COMMIT", &sqlError)) { + rollback(); + return sqlError; + } + + return Void(); +} + +Result openWriterDatabase(const Ref& connection, const ClientSQLOpenRequest& request) { + if (!DiskUtils::isDirectory(request.databasePath.databaseRoot) && + !DiskUtils::makeDirectory(request.databasePath.databaseRoot, true)) { + return Error(STRING_FORMAT( + "Could not create ClientSQLNative database directory '{}'", + request.databasePath.databaseRoot.toString())); + } + + auto writerDatabase = + openSQLiteDatabase(request.databasePath.databasePath, request.name, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, true); + if (!writerDatabase) { + return writerDatabase.moveError(); + } + + auto currentVersion = readUserVersion(writerDatabase.value().get()); + if (!currentVersion) { + return currentVersion.moveError(); + } + + auto schemaResult = + applySchema(writerDatabase.value().get(), + currentVersion.value(), + request.schemaVersion, + request.createStatements, + request.migrations); + if (!schemaResult) { + return schemaResult.moveError(); + } + + connection->database = writerDatabase.moveValue(); + return Void(); +} + +Result openReaderDatabases(const std::vector>& readerConnections, + const ClientSQLOpenRequest& request) { + for (const auto& readerConnection : readerConnections) { + auto readerDatabase = + openSQLiteDatabase(request.databasePath.databasePath, request.name, SQLITE_OPEN_READONLY, false); + if (!readerDatabase) { + auto error = readerDatabase.moveError(); + readerConnection->openError = error; + return error; + } + readerConnection->openError = Error(); + readerConnection->database = readerDatabase.moveValue(); + } + + return Void(); +} + +Result resolveDatabasePath(const Ref& diskCache, const StringBox& name) { + if (name.isEmpty()) { + return Error("ClientSQLNative database name cannot be empty"); + } + + const auto nameView = name.toStringView(); + const auto isLowercaseASCIIAlphaNumeric = [](unsigned char byte) { + return (byte >= 'a' && byte <= 'z') || (byte >= '0' && byte <= '9'); + }; + bool isCanonical = nameView.size() <= 240 && + isLowercaseASCIIAlphaNumeric(static_cast(nameView.front())) && + isLowercaseASCIIAlphaNumeric(static_cast(nameView.back())); + for (const auto character : nameView) { + const auto byte = static_cast(character); + if (!isLowercaseASCIIAlphaNumeric(byte) && byte != '-' && byte != '_' && byte != '.') { + isCanonical = false; + break; + } + } + if (!isCanonical || nameView.find("..") != std::string_view::npos) { + return Error(STRING_FORMAT("Invalid ClientSQLNative database name '{}'", name)); + } + + if (diskCache == nullptr) { + return Error("ClientSQLNative requires an owned database storage root"); + } + + Path storageRoot = diskCache->getRootPath(); + storageRoot.normalize(); + if (storageRoot.empty() || !storageRoot.isAbsolute()) { + return Error("ClientSQLNative database storage root must be nonempty and absolute"); + } + + Path databaseRoot = storageRoot.appending(kClientSQLDirectory); + databaseRoot.normalize(); + + Path databasePath = databaseRoot.appending(name.toStringView()); + databasePath.normalize(); + + if (!databasePath.startsWith(databaseRoot)) { + return Error(STRING_FORMAT("Invalid ClientSQLNative database name '{}'", name)); + } + + return ClientSQLDatabasePath{ + .databasePath = databasePath.toString(), + .databaseRoot = databaseRoot, + }; +} + +} // namespace + +class ClientSQLDatabaseRegistry { +public: + Ref coordinatorForDatabasePath(const std::string& databasePath, + const Ref& fallbackQueue) { + std::lock_guard lock(_mutex); + for (auto iterator = _coordinators.begin(); iterator != _coordinators.end();) { + if (iterator->second.expired()) { + iterator = _coordinators.erase(iterator); + } else { + ++iterator; + } + } + auto existing = _coordinators.find(databasePath); + if (existing != _coordinators.end()) { + auto coordinator = strongRef(existing->second); + if (coordinator != nullptr) { + return coordinator; + } + _coordinators.erase(existing); + } + + auto coordinator = makeShared(fallbackQueue); + _coordinators.emplace(databasePath, coordinator.toWeak()); + return coordinator; + } + +private: + std::mutex _mutex; + std::unordered_map> _coordinators; +}; + +namespace { + +Ref openClientSQLDatabase( + const Ref& diskCache, + const Ref& fallbackQueue, + ClientSQLDatabaseRegistry& databaseRegistry, + StringBox name, + double schemaVersion, + std::vector createStatements, + std::vector migrations) { + if (!std::isfinite(schemaVersion) || std::trunc(schemaVersion) != schemaVersion || + schemaVersion < 1 || + schemaVersion > std::numeric_limits::max()) { + throw Exception("ClientSQLNative schema version must be a positive 32-bit integer"); + } + + std::unordered_set migrationVersions; + for (const auto& migration : migrations) { + const auto rawVersion = static_cast(migration.getVersion()); + if (!std::isfinite(rawVersion) || std::trunc(rawVersion) != rawVersion || rawVersion < 2 || + rawVersion > std::numeric_limits::max()) { + throw Exception("ClientSQLNative migration versions must be 32-bit integers starting at 2"); + } + const auto migrationVersion = static_cast(rawVersion); + if (migrationVersion > static_cast(schemaVersion)) { + throw Exception(STRING_FORMAT( + "ClientSQLNative migration version {} exceeds schema version {}", + migrationVersion, + static_cast(schemaVersion))); + } + if (!migrationVersions.emplace(migrationVersion).second) { + throw Exception(STRING_FORMAT( + "ClientSQLNative duplicate migration version {}", migrationVersion)); + } + } + + auto databasePath = resolveDatabasePath(diskCache, name); + if (!databasePath) { + throw Exception(databasePath.moveError()); + } + + auto databasePathValue = databasePath.moveValue(); + auto coordinator = databaseRegistry.coordinatorForDatabasePath(databasePathValue.databasePath, fallbackQueue); + if (coordinator == nullptr || !coordinator->hasWriterQueue()) { + throw Exception("ClientSQLNative writer queue is unavailable"); + } + + std::vector createStatementValues; + createStatementValues.reserve(createStatements.size()); + for (auto& statement : createStatements) { + createStatementValues.emplace_back(std::move(statement)); + } + + std::vector migrationValues; + migrationValues.reserve(migrations.size()); + for (auto& migration : migrations) { + std::vector statementValues; + statementValues.reserve(migration.getStatements().size()); + for (const auto& statement : migration.getStatements()) { + statementValues.emplace_back(statement); + } + + Value migrationValue; + migrationValue.setMapValue("version", Value(migration.getVersion())); + migrationValue.setMapValue("statements", Value(ValueArray::make(std::move(statementValues)))); + migrationValues.emplace_back(std::move(migrationValue)); + } + + ClientSQLOpenRequest openRequest{ + .name = name, + .databasePath = std::move(databasePathValue), + .schemaVersion = static_cast(schemaVersion), + .createStatements = ValueArray::make(std::move(createStatementValues)), + .migrations = ValueArray::make(std::move(migrationValues)), + }; + auto connection = coordinator->openHandle(std::move(openRequest)); + + return connection; +} + +class ClientSQLNativeModule final : public snap::valdi_modules::client_sql::ClientSQLNativeModule { +public: + ClientSQLNativeModule(Ref diskCache, Ref workerQueue) + : _diskCache(std::move(diskCache)), _workerQueue(std::move(workerQueue)) {} + + Ref openDatabase( + StringBox name, + double schemaVersion, + std::vector createStatements, + std::vector migrations) final { + auto diskCache = _diskCache; + auto workerQueue = _workerQueue; + if (workerQueue == nullptr) { + auto runtime = Runtime::currentRuntime(); + if (runtime == nullptr) { + throw Exception("ClientSQLNative requires an active Valdi runtime"); + } + diskCache = runtime->getDiskCache(); + workerQueue = runtime->getWorkerQueue(); + } + return openClientSQLDatabase(diskCache, + workerQueue, + _databaseRegistry, + std::move(name), + schemaVersion, + std::move(createStatements), + std::move(migrations)); + } + +private: + Ref _diskCache; + Ref _workerQueue; + ClientSQLDatabaseRegistry _databaseRegistry; +}; + +} // namespace + +ClientSQLNativeModuleFactory::ClientSQLNativeModuleFactory() = default; + +ClientSQLNativeModuleFactory::ClientSQLNativeModuleFactory(const Ref& diskCache, + const Ref& workerQueue) + : _diskCache(diskCache), _workerQueue(workerQueue) {} + +ClientSQLNativeModuleFactory::~ClientSQLNativeModuleFactory() = default; + +Ref ClientSQLNativeModuleFactory::onLoadModule() { + return makeShared(_diskCache, _workerQueue); +} + +static auto kRegisterModule = Valdi::RegisterModuleFactory::registerTyped(); + +} // namespace Valdi diff --git a/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.hpp b/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.hpp new file mode 100644 index 000000000..bfdd1d71b --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "valdi/runtime/Interfaces/IDiskCache.hpp" +#include "valdi_modules/client_sql/client_sql.hpp" +#include "valdi_core/cpp/Threading/IDispatchQueue.hpp" +#include "valdi_core/cpp/Utils/Shared.hpp" + +namespace Valdi { + +class ClientSQLNativeModuleFactory : public snap::valdi_modules::client_sql::ClientSQLNativeModuleFactory { +public: + ClientSQLNativeModuleFactory(); + ClientSQLNativeModuleFactory(const Ref& diskCache, const Ref& workerQueue); + ~ClientSQLNativeModuleFactory() override; + +protected: + Ref onLoadModule() override; + +private: + Ref _diskCache; + Ref _workerQueue; +}; + +} // namespace Valdi diff --git a/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory_tests.cpp b/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory_tests.cpp new file mode 100644 index 000000000..9c1d291db --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/native/ClientSQLNativeModuleFactory_tests.cpp @@ -0,0 +1,1274 @@ +#include "ClientSQLNativeModuleFactory.hpp" +#include "valdi/runtime/Resources/DiskCacheImpl.hpp" +#include "valdi_core/cpp/Threading/DispatchQueue.hpp" +#include "valdi_core/cpp/Utils/DiskUtils.hpp" +#include "valdi_core/cpp/Utils/Exception.hpp" +#include "valdi_core/cpp/Utils/Format.hpp" +#include "valdi_core/cpp/Utils/ValueArray.hpp" +#include "valdi_core/cpp/Utils/ValueFunctionWithCallable.hpp" +#include "valdi_core/cpp/Utils/ValueTypedArray.hpp" +#include "valdi_core/cpp/Utils/ValueTypedProxyObject.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Valdi; + +namespace ValdiTest { + +class ClientSQLTemporaryDirectory { +public: + ClientSQLTemporaryDirectory() { + char directoryLocation[] = "/tmp/.valdi_clientsql_test.XXXXXX"; + if (mkdtemp(directoryLocation) == nullptr) { + throw Exception(STRING_FORMAT("Failed to create temporary directory: {}", strerror(errno))); + } + _rootDirectory = STRING_LITERAL(directoryLocation); + } + + ~ClientSQLTemporaryDirectory() { + if (!DiskUtils::remove(Path(_rootDirectory.toStringView()))) { + std::cout << "Failed to delete temporary directory: " << strerror(errno) << std::endl; + } + } + + const StringBox& get() const { + return _rootDirectory; + } + +private: + StringBox _rootDirectory; +}; + +class ClientSQLAsyncResult { +public: + Ref makeCallback() { + return makeShared([this](const ValueFunctionCallContext& callContext) -> Value { + { + std::lock_guard lock(_mutex); + _value = callContext.getParameter(0); + _error = callContext.getParameter(1); + _callbackThreadId = std::this_thread::get_id(); + _called = true; + } + _condition.notify_one(); + return Value::undefined(); + }); + } + + Value waitForSuccess() { + return waitForSuccessWithin(std::chrono::seconds(5)); + } + + Value waitForSuccessWithin(std::chrono::milliseconds timeout) { + std::unique_lock lock(_mutex); + if (!_condition.wait_for(lock, timeout, [this]() { return _called; })) { + ADD_FAILURE() << "Timed out waiting for ClientSQL callback"; + return Value::undefined(); + } + EXPECT_TRUE(_error.isNullOrUndefined()) << _error.toString(); + return _value; + } + + Value waitForError() { + std::unique_lock lock(_mutex); + if (!_condition.wait_for(lock, std::chrono::seconds(5), [this]() { return _called; })) { + ADD_FAILURE() << "Timed out waiting for ClientSQL callback"; + return Value::undefined(); + } + EXPECT_TRUE(_value.isNullOrUndefined()) << _value.toString(); + EXPECT_FALSE(_error.isNullOrUndefined()); + return _error; + } + + bool wasCalled() const { + std::lock_guard lock(_mutex); + return _called; + } + + std::thread::id getCallbackThreadId() const { + std::lock_guard lock(_mutex); + return _callbackThreadId; + } + +private: + mutable std::mutex _mutex; + std::condition_variable _condition; + bool _called = false; + Value _value = Value::undefined(); + Value _error = Value::undefined(); + std::thread::id _callbackThreadId; +}; + +Value callNativeFunction(const Value& object, std::string_view method, const std::vector& parameters) { + auto callableObject = object; + if (object.isTypedObject()) { + callableObject = Value(object.getTypedObjectRef()->toValueMap(true)); + } else if (object.isProxyObject()) { + callableObject = Value(object.getTypedProxyObjectRef()->getTypedObject()->toValueMap(true)); + } + + auto function = callableObject.getMapValue(method).getFunctionRef(); + EXPECT_NE(function, nullptr) << method << ": " << callableObject.toString(); + if (function == nullptr) { + return Value::undefined(); + } + + auto result = (*function)(parameters.data(), parameters.size()); + EXPECT_TRUE(result.success()) << result.description(); + if (!result) { + return Value::undefined(); + } + return result.moveValue(); +} + +std::string callNativeFunctionExpectingError(const Value& object, + std::string_view method, + const std::vector& parameters) { + auto callableObject = object; + if (object.isTypedObject()) { + callableObject = Value(object.getTypedObjectRef()->toValueMap(true)); + } else if (object.isProxyObject()) { + callableObject = Value(object.getTypedProxyObjectRef()->getTypedObject()->toValueMap(true)); + } + + auto function = callableObject.getMapValue(method).getFunctionRef(); + EXPECT_NE(function, nullptr) << method << ": " << callableObject.toString(); + if (function == nullptr) { + return "Native function is unavailable"; + } + + auto result = (*function)(parameters.data(), parameters.size()); + EXPECT_TRUE(result.failure()) << result.description(); + return result.description(); +} + +std::vector makeClientSQLOpenParameters(StringBox name) { + return { + Value(std::move(name)), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY)")})), + Value(ValueArray::make(0)), + }; +} + +Value callClientSQLAsync(const Value& connection, std::string_view method, StringBox sql, Value parameters) { + ClientSQLAsyncResult asyncResult; + std::vector callParameters{ + Value(sql), + std::move(parameters), + Value(asyncResult.makeCallback()), + }; + + callNativeFunction(connection, method, callParameters); + return asyncResult.waitForSuccess(); +} + +Value queryClientSQL(const Value& connection, + std::string_view method, + StringBox sql, + Value parameters) { + return callClientSQLAsync(connection, method, sql, std::move(parameters)); +} + +void executeClientSQL(const Value& connection, StringBox sql, Value parameters) { + callClientSQLAsync(connection, "execute", sql, std::move(parameters)); +} + +TEST(ClientSQLNativeModuleFactory, rejectsTraversalDatabaseName) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + const auto error = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("../escape.sqlite"))); + EXPECT_NE(std::string::npos, error.find("Invalid ClientSQLNative database name")) << error; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsAbsoluteDatabaseName) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + const auto error = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("/tmp/escape.sqlite"))); + EXPECT_NE(std::string::npos, error.find("Invalid ClientSQLNative database name")) << error; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsMissingDatabaseStorageRoot) { + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(nullptr, workerQueue); + auto module = factory->loadModule(); + + const auto error = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("missing-root.sqlite"))); + EXPECT_NE(std::string::npos, error.find("requires an owned database storage root")) << error; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsControlCharactersInDatabaseName) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + const auto embeddedNullName = StringBox::fromString( + std::string_view("nul\0name.sqlite", sizeof("nul\0name.sqlite") - 1)); + const auto nullError = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(embeddedNullName)); + EXPECT_NE(std::string::npos, nullError.find("Invalid ClientSQLNative database name")) << nullError; + + const auto controlError = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(StringBox::fromString(std::string_view("line\nfeed.sqlite")))); + EXPECT_NE(std::string::npos, controlError.find("Invalid ClientSQLNative database name")) << controlError; + + const auto c1ControlError = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(StringBox::fromString(std::string_view("next\xc2\x85line.sqlite")))); + EXPECT_NE(std::string::npos, c1ControlError.find("Invalid ClientSQLNative database name")) << c1ControlError; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsFilesystemAliasDatabaseNames) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + for (const auto& name : {"Uppercase.sqlite", "trailing-dot.", "double..dot.sqlite", "trailing-space.sqlite "}) { + const auto error = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(StringBox::fromString(std::string_view(name)))); + EXPECT_NE(std::string::npos, error.find("Invalid ClientSQLNative database name")) << name << ": " << error; + } + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsEmptyDatabaseStorageRoot) { + auto diskCache = makeShared(StringBox::emptyString()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + const auto error = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("empty-root.sqlite"))); + EXPECT_NE(std::string::npos, error.find("database storage root must be nonempty and absolute")) << error; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsRelativeDatabaseStorageRoot) { + auto diskCache = makeShared(STRING_LITERAL("relative-root")); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + const auto error = callNativeFunctionExpectingError( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("relative-root.sqlite"))); + EXPECT_NE(std::string::npos, error.find("database storage root must be nonempty and absolute")) << error; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, opensWALDatabaseAndUsesReadonlyReaders) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + executeClientSQL(connection, + STRING_LITERAL("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Ada")}))); + + auto rows = queryClientSQL( + connection, "query", STRING_LITERAL("SELECT name FROM item ORDER BY id"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_EQ(STRING_LITERAL("Ada"), (*rows)[0].getMapValue("name").toStringBox()); + + auto journalRows = queryClientSQL( + connection, "queryOnWriter", STRING_LITERAL("PRAGMA journal_mode"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(journalRows, nullptr); + ASSERT_EQ(1ul, journalRows->size()); + EXPECT_EQ(STRING_LITERAL("wal"), (*journalRows)[0].getMapValue("journal_mode").toStringBox()); + + auto readerRows = + queryClientSQL(connection, "query", STRING_LITERAL("PRAGMA query_only"), Value::undefined()).getArrayRef(); + ASSERT_NE(readerRows, nullptr); + ASSERT_EQ(1ul, readerRows->size()); + EXPECT_EQ(1, (*readerRows)[0].getMapValue("query_only").toInt()); + + ClientSQLAsyncResult debugInfoResult; + callNativeFunction(connection, "debugInfo", {Value(debugInfoResult.makeCallback())}); + const auto debugInfo = debugInfoResult.waitForSuccess(); + EXPECT_GE(debugInfo.getMapValue("sqliteVersionNumber").toInt(), 3016000); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, dispatchesWriterWorkOffCallerThread) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-thread-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + ClientSQLAsyncResult insertResult; + callNativeFunction(connection, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Ada")})), + Value(insertResult.makeCallback())}); + + insertResult.waitForSuccess(); + EXPECT_NE(std::this_thread::get_id(), insertResult.getCallbackThreadId()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsIntegersOutsideJavaScriptSafeRange) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-integer-range-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + ClientSQLAsyncResult queryResult; + callNativeFunction(connection, + "query", + {Value("SELECT 9007199254740992 AS value"), + Value(ValueArray::make(0)), + Value(queryResult.makeCallback())}); + const auto error = queryResult.waitForError().toString(); + EXPECT_NE(std::string::npos, error.find("exceeds JavaScript's exact integer range")) << error; + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsNonFiniteDoubleParameters) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + auto connection = callNativeFunction( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("runtime-nonfinite-test.sqlite"))); + + ClientSQLAsyncResult infinityResult; + callNativeFunction(connection, + "query", + {Value("SELECT ? AS value"), + Value(ValueArray::make({Value(std::numeric_limits::infinity())})), + Value(infinityResult.makeCallback())}); + const auto infinityError = infinityResult.waitForError().toString(); + EXPECT_NE(std::string::npos, infinityError.find("double parameters must be finite")) << infinityError; + + ClientSQLAsyncResult nanResult; + callNativeFunction(connection, + "query", + {Value("SELECT ? AS value"), + Value(ValueArray::make({Value(std::numeric_limits::quiet_NaN())})), + Value(nanResult.makeCallback())}); + const auto nanError = nanResult.waitForError().toString(); + EXPECT_NE(std::string::npos, nanError.find("double parameters must be finite")) << nanError; + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, readsEmptyTextAndBlobValuesSafely) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + auto connection = callNativeFunction( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("runtime-empty-values-test.sqlite"))); + + auto rows = queryClientSQL( + connection, + "query", + STRING_LITERAL("SELECT '' AS empty_text, CAST(X'' AS BLOB) AS empty_blob"), + Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_TRUE((*rows)[0].getMapValue("empty_text").toStringBox().isEmpty()); + const auto& emptyBlob = (*rows)[0].getMapValue("empty_blob"); + ASSERT_EQ(ValueType::TypedArray, emptyBlob.getType()); + ASSERT_NE(nullptr, emptyBlob.getTypedArray()); + EXPECT_EQ(0ul, emptyBlob.getTypedArray()->getBuffer().size()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, bindsEmptyArrayBufferAsZeroLengthBlob) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + std::vector openParameters{ + Value("runtime-empty-blob-bind-test.sqlite"), + Value(1), + Value(ValueArray::make({Value( + "CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, payload BLOB NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + auto bytes = makeShared(); + auto emptyBlob = Value(makeShared(TypedArrayType::ArrayBuffer, bytes)); + executeClientSQL( + connection, + STRING_LITERAL("INSERT INTO item(id, payload) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), std::move(emptyBlob)}))); + + auto rows = queryClientSQL( + connection, + "query", + STRING_LITERAL( + "SELECT typeof(payload) AS storage_type, length(payload) AS payload_length, payload FROM item"), + Value(ValueArray::make(0))) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_EQ(STRING_LITERAL("blob"), (*rows)[0].getMapValue("storage_type").toStringBox()); + EXPECT_EQ(0, (*rows)[0].getMapValue("payload_length").toInt()); + const auto& roundTrippedBlob = (*rows)[0].getMapValue("payload"); + ASSERT_EQ(ValueType::TypedArray, roundTrippedBlob.getType()); + EXPECT_EQ(0ul, roundTrippedBlob.getTypedArray()->getBuffer().size()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsWritesThroughReadonlyReaders) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-reader-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + executeClientSQL(connection, + STRING_LITERAL("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Ada")}))); + + auto readerRows = + queryClientSQL(connection, "query", STRING_LITERAL("PRAGMA query_only"), Value::undefined()).getArrayRef(); + ASSERT_NE(readerRows, nullptr); + ASSERT_EQ(1ul, readerRows->size()); + EXPECT_EQ(1, (*readerRows)[0].getMapValue("query_only").toInt()); + + ClientSQLAsyncResult readerWriteResult; + callNativeFunction(connection, + "query", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(2), Value("Grace")})), + Value(readerWriteResult.makeCallback())}); + + const auto error = readerWriteResult.waitForError().toString(); + EXPECT_NE(std::string::npos, error.find("attempt to write a readonly database")) << error; + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, keepsSharedDatabaseOpenUntilLastHandleCloses) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-shared-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto firstConnection = callNativeFunction(module, "openDatabase", openParameters); + auto secondConnection = callNativeFunction(module, "openDatabase", openParameters); + + executeClientSQL(firstConnection, + STRING_LITERAL("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Ada")}))); + + ClientSQLAsyncResult firstCloseResult; + callNativeFunction(firstConnection, "close", {Value(firstCloseResult.makeCallback())}); + firstCloseResult.waitForSuccess(); + + executeClientSQL(secondConnection, + STRING_LITERAL("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(2), Value("Grace")}))); + + auto rows = queryClientSQL( + secondConnection, "query", STRING_LITERAL("SELECT name FROM item ORDER BY id"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(2ul, rows->size()); + EXPECT_EQ(STRING_LITERAL("Ada"), (*rows)[0].getMapValue("name").toStringBox()); + EXPECT_EQ(STRING_LITERAL("Grace"), (*rows)[1].getMapValue("name").toStringBox()); + + ClientSQLAsyncResult secondCloseResult; + callNativeFunction(secondConnection, "close", {Value(secondCloseResult.makeCallback())}); + secondCloseResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, serializesWritesAcrossSharedHandles) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-shared-writer-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto firstConnection = callNativeFunction(module, "openDatabase", openParameters); + auto secondConnection = callNativeFunction(module, "openDatabase", openParameters); + + constexpr int writeCount = 20; + std::vector> writeResults; + writeResults.reserve(writeCount); + for (int index = 0; index < writeCount; ++index) { + auto result = std::make_unique(); + const auto& connection = index % 2 == 0 ? firstConnection : secondConnection; + callNativeFunction(connection, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(index + 1), Value(STRING_FORMAT("Item {}", index + 1))})), + Value(result->makeCallback())}); + writeResults.emplace_back(std::move(result)); + } + + for (const auto& result : writeResults) { + result->waitForSuccess(); + } + + auto rows = queryClientSQL( + firstConnection, "query", STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_EQ(writeCount, (*rows)[0].getMapValue("count").toInt()); + + ClientSQLAsyncResult firstCloseResult; + callNativeFunction(firstConnection, "close", {Value(firstCloseResult.makeCallback())}); + firstCloseResult.waitForSuccess(); + + ClientSQLAsyncResult secondCloseResult; + callNativeFunction(secondConnection, "close", {Value(secondCloseResult.makeCallback())}); + secondCloseResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, commitsNativeTransactionAfterDoneCallback) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-transaction-commit-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + auto body = makeShared([](const ValueFunctionCallContext& callContext) -> Value { + auto transaction = callContext.getParameter(0); + auto doneCallback = callContext.getParameterAsFunction(1); + if (!callContext.getExceptionTracker()) { + return Value::undefined(); + } + + auto insertCallback = makeShared( + [transaction, doneCallback](const ValueFunctionCallContext& insertContext) -> Value { + const auto& insertError = insertContext.getParameter(1); + if (!insertError.isNullOrUndefined()) { + (*doneCallback)({Value::undefined(), insertError}); + return Value::undefined(); + } + + auto queryCallback = makeShared( + [doneCallback](const ValueFunctionCallContext& queryContext) -> Value { + const auto& queryError = queryContext.getParameter(1); + if (!queryError.isNullOrUndefined()) { + (*doneCallback)({Value::undefined(), queryError}); + return Value::undefined(); + } + + auto rows = queryContext.getParameter(0).getArrayRef(); + if (rows == nullptr || rows->size() != 1 || + (*rows)[0].getMapValue("count").toInt() != 1) { + (*doneCallback)({Value::undefined(), Value("transaction query did not observe write")}); + return Value::undefined(); + } + + (*doneCallback)({Value("committed"), Value::undefined()}); + return Value::undefined(); + }); + callNativeFunction( + transaction, + "query", + {Value("SELECT COUNT(*) AS count FROM item"), Value(ValueArray::make(0)), Value(queryCallback)}); + return Value::undefined(); + }); + callNativeFunction( + transaction, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Ada")})), + Value(insertCallback)}); + return Value::undefined(); + }); + + ClientSQLAsyncResult transactionResult; + callNativeFunction(connection, "transaction", {Value(body), Value(transactionResult.makeCallback())}); + EXPECT_EQ(STRING_LITERAL("committed"), transactionResult.waitForSuccess().toStringBox()); + + auto rows = queryClientSQL( + connection, "query", STRING_LITERAL("SELECT name FROM item ORDER BY id"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_EQ(STRING_LITERAL("Ada"), (*rows)[0].getMapValue("name").toStringBox()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rollsBackNativeTransactionWhenDoneReceivesError) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-transaction-rollback-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto connection = callNativeFunction(module, "openDatabase", openParameters); + + auto body = makeShared([](const ValueFunctionCallContext& callContext) -> Value { + auto transaction = callContext.getParameter(0); + auto doneCallback = callContext.getParameterAsFunction(1); + if (!callContext.getExceptionTracker()) { + return Value::undefined(); + } + + auto insertCallback = makeShared( + [doneCallback](const ValueFunctionCallContext& insertContext) -> Value { + const auto& insertError = insertContext.getParameter(1); + if (!insertError.isNullOrUndefined()) { + (*doneCallback)({Value::undefined(), insertError}); + return Value::undefined(); + } + + (*doneCallback)({Value::undefined(), Value("rollback requested")}); + return Value::undefined(); + }); + callNativeFunction( + transaction, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Ada")})), + Value(insertCallback)}); + return Value::undefined(); + }); + + ClientSQLAsyncResult transactionResult; + callNativeFunction(connection, "transaction", {Value(body), Value(transactionResult.makeCallback())}); + const auto error = transactionResult.waitForError().toString(); + EXPECT_NE(std::string::npos, error.find("rollback requested")) << error; + + auto rows = queryClientSQL( + connection, "query", STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_EQ(0, (*rows)[0].getMapValue("count").toInt()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, defersExternalWriterWorkUntilNativeTransactionFinishes) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-transaction-queue-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto firstConnection = callNativeFunction(module, "openDatabase", openParameters); + auto secondConnection = callNativeFunction(module, "openDatabase", openParameters); + + std::mutex mutex; + std::condition_variable condition; + bool transactionReady = false; + Ref savedDoneCallback; + + auto body = makeShared( + [&mutex, &condition, &transactionReady, &savedDoneCallback](const ValueFunctionCallContext& callContext) + -> Value { + auto transaction = callContext.getParameter(0); + auto doneCallback = callContext.getParameterAsFunction(1); + if (!callContext.getExceptionTracker()) { + return Value::undefined(); + } + + auto insertCallback = makeShared( + [&mutex, &condition, &transactionReady, &savedDoneCallback, doneCallback]( + const ValueFunctionCallContext& insertContext) -> Value { + const auto& insertError = insertContext.getParameter(1); + if (!insertError.isNullOrUndefined()) { + (*doneCallback)({Value::undefined(), insertError}); + return Value::undefined(); + } + + { + std::lock_guard lock(mutex); + savedDoneCallback = doneCallback; + transactionReady = true; + } + condition.notify_one(); + return Value::undefined(); + }); + callNativeFunction( + transaction, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Inside")})), + Value(insertCallback)}); + return Value::undefined(); + }); + + ClientSQLAsyncResult transactionResult; + callNativeFunction(firstConnection, "transaction", {Value(body), Value(transactionResult.makeCallback())}); + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(condition.wait_for(lock, std::chrono::seconds(5), [&transactionReady]() { + return transactionReady; + })); + } + + ClientSQLAsyncResult outsideWriteResult; + callNativeFunction(secondConnection, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(2), Value("Outside")})), + Value(outsideWriteResult.makeCallback())}); + + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + EXPECT_FALSE(outsideWriteResult.wasCalled()); + + ClientSQLAsyncResult debugInfoResult; + callNativeFunction(firstConnection, "debugInfo", {Value(debugInfoResult.makeCallback())}); + auto debugInfo = debugInfoResult.waitForSuccess(); + EXPECT_TRUE(debugInfo.getMapValue("activeTransaction").toBool()); + EXPECT_GE(debugInfo.getMapValue("deferredWriterWork").toInt(), 1); + EXPECT_EQ(2, debugInfo.getMapValue("activeHandles").toInt()); + + Ref doneCallback; + { + std::lock_guard lock(mutex); + doneCallback = savedDoneCallback; + } + ASSERT_NE(doneCallback, nullptr); + (*doneCallback)({Value::undefined(), Value::undefined()}); + + transactionResult.waitForSuccess(); + outsideWriteResult.waitForSuccess(); + + auto rows = queryClientSQL( + firstConnection, "query", STRING_LITERAL("SELECT name FROM item ORDER BY id"), Value::undefined()) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(2ul, rows->size()); + EXPECT_EQ(STRING_LITERAL("Inside"), (*rows)[0].getMapValue("name").toStringBox()); + EXPECT_EQ(STRING_LITERAL("Outside"), (*rows)[1].getMapValue("name").toStringBox()); + + ClientSQLAsyncResult firstCloseResult; + callNativeFunction(firstConnection, "close", {Value(firstCloseResult.makeCallback())}); + firstCloseResult.waitForSuccess(); + + ClientSQLAsyncResult secondCloseResult; + callNativeFunction(secondConnection, "close", {Value(secondCloseResult.makeCallback())}); + secondCloseResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, allowsSeparateHandleSnapshotReadersDuringActiveNativeTransaction) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector openParameters{ + Value("runtime-transaction-reader-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL)")})), + Value(ValueArray::make(0)), + }; + auto firstConnection = callNativeFunction(module, "openDatabase", openParameters); + auto secondConnection = callNativeFunction(module, "openDatabase", openParameters); + + executeClientSQL(firstConnection, + STRING_LITERAL("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(1), Value("Before")}))); + + std::mutex mutex; + std::condition_variable condition; + bool transactionReady = false; + Ref savedDoneCallback; + + auto body = makeShared( + [&mutex, &condition, &transactionReady, &savedDoneCallback](const ValueFunctionCallContext& callContext) + -> Value { + auto transaction = callContext.getParameter(0); + auto doneCallback = callContext.getParameterAsFunction(1); + if (!callContext.getExceptionTracker()) { + return Value::undefined(); + } + + auto insertCallback = makeShared( + [&mutex, &condition, &transactionReady, &savedDoneCallback, doneCallback]( + const ValueFunctionCallContext& insertContext) -> Value { + const auto& insertError = insertContext.getParameter(1); + if (!insertError.isNullOrUndefined()) { + (*doneCallback)({Value::undefined(), insertError}); + return Value::undefined(); + } + + { + std::lock_guard lock(mutex); + savedDoneCallback = doneCallback; + transactionReady = true; + } + condition.notify_one(); + return Value::undefined(); + }); + callNativeFunction( + transaction, + "execute", + {Value("INSERT INTO item(id, name) VALUES (?, ?)"), + Value(ValueArray::make({Value(2), Value("Inside")})), + Value(insertCallback)}); + return Value::undefined(); + }); + + ClientSQLAsyncResult transactionResult; + callNativeFunction(firstConnection, "transaction", {Value(body), Value(transactionResult.makeCallback())}); + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(condition.wait_for(lock, std::chrono::seconds(5), [&transactionReady]() { + return transactionReady; + })); + } + + ClientSQLAsyncResult outsideReadResult; + callNativeFunction(secondConnection, + "query", + {Value("SELECT name FROM item ORDER BY id"), + Value(ValueArray::make(0)), + Value(outsideReadResult.makeCallback())}); + + auto rowsBeforeCommit = outsideReadResult.waitForSuccessWithin(std::chrono::seconds(1)).getArrayRef(); + ASSERT_NE(rowsBeforeCommit, nullptr); + ASSERT_EQ(1ul, rowsBeforeCommit->size()); + EXPECT_EQ(STRING_LITERAL("Before"), (*rowsBeforeCommit)[0].getMapValue("name").toStringBox()); + + Ref doneCallback; + { + std::lock_guard lock(mutex); + doneCallback = savedDoneCallback; + } + ASSERT_NE(doneCallback, nullptr); + (*doneCallback)({Value::undefined(), Value::undefined()}); + + transactionResult.waitForSuccess(); + + auto rowsAfterCommit = queryClientSQL( + secondConnection, + "query", + STRING_LITERAL("SELECT name FROM item ORDER BY id"), + Value::undefined()) + .getArrayRef(); + ASSERT_NE(rowsAfterCommit, nullptr); + ASSERT_EQ(2ul, rowsAfterCommit->size()); + EXPECT_EQ(STRING_LITERAL("Before"), (*rowsAfterCommit)[0].getMapValue("name").toStringBox()); + EXPECT_EQ(STRING_LITERAL("Inside"), (*rowsAfterCommit)[1].getMapValue("name").toStringBox()); + + ClientSQLAsyncResult firstCloseResult; + callNativeFunction(firstConnection, "close", {Value(firstCloseResult.makeCallback())}); + firstCloseResult.waitForSuccess(); + + ClientSQLAsyncResult secondCloseResult; + callNativeFunction(secondConnection, "close", {Value(secondCloseResult.makeCallback())}); + secondCloseResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsParentConnectionReentrancyInsideTransactionBody) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + auto connection = callNativeFunction( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("runtime-transaction-reentrancy-test.sqlite"))); + + std::atomic_bool executeRejected{false}; + std::atomic_bool queryRejected{false}; + std::atomic_bool closeRejected{false}; + auto body = makeShared( + [connection, &executeRejected, &queryRejected, &closeRejected]( + const ValueFunctionCallContext& callContext) -> Value { + auto doneCallback = callContext.getParameterAsFunction(1); + if (!callContext.getExceptionTracker()) { + return Value::undefined(); + } + + auto executeCallback = makeShared( + [&executeRejected](const ValueFunctionCallContext& executeContext) -> Value { + executeRejected.store( + executeContext.getParameter(1).toString().find("parent connection execute") != std::string::npos); + return Value::undefined(); + }); + callNativeFunction( + connection, + "execute", + {Value("INSERT INTO item(id) VALUES (1)"), + Value(ValueArray::make(0)), + Value(executeCallback)}); + + auto queryCallback = makeShared( + [&queryRejected](const ValueFunctionCallContext& queryContext) -> Value { + queryRejected.store( + queryContext.getParameter(1).toString().find("parent connection query") != std::string::npos); + return Value::undefined(); + }); + callNativeFunction( + connection, + "query", + {Value("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0)), + Value(queryCallback)}); + + auto closeCallback = makeShared( + [&closeRejected, doneCallback](const ValueFunctionCallContext& closeContext) -> Value { + closeRejected.store( + closeContext.getParameter(1).toString().find("parent connection close") != std::string::npos); + (*doneCallback)({Value::undefined(), Value::undefined()}); + return Value::undefined(); + }); + callNativeFunction(connection, "close", {Value(closeCallback)}); + return Value::undefined(); + }); + + ClientSQLAsyncResult transactionResult; + callNativeFunction(connection, "transaction", {Value(body), Value(transactionResult.makeCallback())}); + transactionResult.waitForSuccess(); + EXPECT_TRUE(executeRejected.load()); + EXPECT_TRUE(queryRejected.load()); + EXPECT_TRUE(closeRejected.load()); + + auto rows = queryClientSQL( + connection, + "query", + STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0))) + .getArrayRef(); + ASSERT_NE(rows, nullptr); + ASSERT_EQ(1ul, rows->size()); + EXPECT_EQ(0, (*rows)[0].getMapValue("count").toInt()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsSchemaFingerprintMismatchForSharedIdentity) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + auto firstConnection = callNativeFunction( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("schema-fingerprint-test.sqlite"))); + queryClientSQL( + firstConnection, + "query", + STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0))); + + std::vector collidingOpenParameters{ + Value("schema-fingerprint-test.sqlite"), + Value(1), + Value(ValueArray::make({ + Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY)"), + Value("SELECT 'schema-boundary-a'"), + })), + Value(ValueArray::make(0)), + }; + auto collidingConnection = callNativeFunction(module, "openDatabase", collidingOpenParameters); + ClientSQLAsyncResult queryResult; + callNativeFunction( + collidingConnection, + "query", + {Value("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0)), + Value(queryResult.makeCallback())}); + const auto error = queryResult.waitForError().toString(); + EXPECT_NE(std::string::npos, error.find("different schema fingerprint")) << error; + + ClientSQLAsyncResult firstCloseResult; + callNativeFunction(firstConnection, "close", {Value(firstCloseResult.makeCallback())}); + firstCloseResult.waitForSuccess(); + ClientSQLAsyncResult collisionCloseResult; + callNativeFunction(collidingConnection, "close", {Value(collisionCloseResult.makeCallback())}); + collisionCloseResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, releasesManyClosedDatabaseCoordinators) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + for (int index = 0; index < 64; ++index) { + auto connection = callNativeFunction( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_FORMAT("lifetime-{}.sqlite", index))); + queryClientSQL( + connection, + "query", + STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0))); + ClientSQLAsyncResult closeResult; + callNativeFunction(connection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + } + + auto finalConnection = callNativeFunction( + module, + "openDatabase", + makeClientSQLOpenParameters(STRING_LITERAL("lifetime-final.sqlite"))); + queryClientSQL( + finalConnection, + "query", + STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0))); + ClientSQLAsyncResult debugInfoResult; + callNativeFunction(finalConnection, "debugInfo", {Value(debugInfoResult.makeCallback())}); + EXPECT_EQ(1, debugInfoResult.waitForSuccess().getMapValue("liveCoordinators").toInt()); + + ClientSQLAsyncResult closeResult; + callNativeFunction(finalConnection, "close", {Value(closeResult.makeCallback())}); + closeResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, validatesSchemaAndMigrationVersionsAtBoundary) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + auto invalidSchemaParameters = makeClientSQLOpenParameters(STRING_LITERAL("invalid-schema-version.sqlite")); + invalidSchemaParameters[1] = Value(0); + const auto schemaError = callNativeFunctionExpectingError(module, "openDatabase", invalidSchemaParameters); + EXPECT_NE(std::string::npos, schemaError.find("positive 32-bit integer")) << schemaError; + + Value firstMigration; + firstMigration.setMapValue("version", Value(2)); + firstMigration.setMapValue("statements", Value(ValueArray::make({Value("SELECT 1")}))); + Value duplicateMigration; + duplicateMigration.setMapValue("version", Value(2)); + duplicateMigration.setMapValue("statements", Value(ValueArray::make({Value("SELECT 2")}))); + std::vector duplicateMigrationParameters{ + Value("duplicate-migration-version.sqlite"), + Value(2), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY)")})), + Value(ValueArray::make({firstMigration, duplicateMigration})), + }; + const auto duplicateError = + callNativeFunctionExpectingError(module, "openDatabase", duplicateMigrationParameters); + EXPECT_NE(std::string::npos, duplicateError.find("duplicate migration version 2")) << duplicateError; + workerQueue->fullTeardown(); +} + +TEST(ClientSQLNativeModuleFactory, rejectsMissingMigrationVersions) { + ClientSQLTemporaryDirectory directory; + auto diskCache = makeShared(directory.get()); + auto workerQueue = DispatchQueue::create(STRING_LITERAL("ClientSQL Test Worker"), ThreadQoSClassNormal); + auto factory = makeShared(diskCache, workerQueue); + auto module = factory->loadModule(); + + std::vector initialOpenParameters{ + Value("runtime-migration-gap-test.sqlite"), + Value(1), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY)")})), + Value(ValueArray::make(0)), + }; + auto initialConnection = callNativeFunction(module, "openDatabase", initialOpenParameters); + queryClientSQL( + initialConnection, "query", STRING_LITERAL("SELECT COUNT(*) AS count FROM item"), Value::undefined()); + + ClientSQLAsyncResult initialCloseResult; + callNativeFunction(initialConnection, "close", {Value(initialCloseResult.makeCallback())}); + initialCloseResult.waitForSuccess(); + + Value versionThreeMigration; + versionThreeMigration.setMapValue("version", Value(3)); + versionThreeMigration.setMapValue( + "statements", + Value(ValueArray::make({Value("ALTER TABLE item ADD COLUMN name TEXT")}))); + std::vector upgradedOpenParameters{ + Value("runtime-migration-gap-test.sqlite"), + Value(3), + Value(ValueArray::make({Value("CREATE TABLE item (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")})), + Value(ValueArray::make({versionThreeMigration})), + }; + auto upgradedConnection = callNativeFunction(module, "openDatabase", upgradedOpenParameters); + + ClientSQLAsyncResult queryResult; + callNativeFunction(upgradedConnection, + "query", + {Value("SELECT COUNT(*) AS count FROM item"), + Value(ValueArray::make(0)), + Value(queryResult.makeCallback())}); + const auto error = queryResult.waitForError().toString(); + EXPECT_NE(std::string::npos, error.find("missing migration for schema version 2")) << error; + + ClientSQLAsyncResult upgradedCloseResult; + callNativeFunction(upgradedConnection, "close", {Value(upgradedCloseResult.makeCallback())}); + upgradedCloseResult.waitForSuccess(); + workerQueue->fullTeardown(); +} + +} // namespace ValdiTest diff --git a/src/valdi_modules/src/valdi/client_sql/src/ClientSQL.ts b/src/valdi_modules/src/valdi/client_sql/src/ClientSQL.ts new file mode 100644 index 000000000..967cff6c2 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/src/ClientSQL.ts @@ -0,0 +1,5 @@ +import { ClientSQLNativeModule, openDatabase } from './ClientSQLNative'; + +export const clientSQLNative: ClientSQLNativeModule = { + openDatabase: openDatabase as unknown as ClientSQLNativeModule['openDatabase'], +}; diff --git a/src/valdi_modules/src/valdi/client_sql/src/ClientSQLDebug.ts b/src/valdi_modules/src/valdi/client_sql/src/ClientSQLDebug.ts new file mode 100644 index 000000000..3c438e7de --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/src/ClientSQLDebug.ts @@ -0,0 +1,928 @@ +import { + createDebuggerProviderOwner, + createDebuggerProviderResult, + DebuggerProviderKind, +} from 'valdi_core/src/debugging/DebuggerProvider'; +import type { + DebuggerProviderModule, + DebuggerProviderOwner, + DebuggerProviderRegistration, + DebuggerProviderRequest, + DebuggerProviderResult, +} from 'valdi_core/src/debugging/DebuggerProvider'; + +declare const module: DebuggerProviderModule; + +export type ClientSQLDebugValue = string | number | boolean | ArrayBuffer | null; + +export interface ClientSQLDebugColumn { + cid: number; + name: string; + type: string; + notnull: number; + dflt_value: unknown; + pk: number; +} + +export interface ClientSQLDebugDatabase { + id: string; + name: string; + schemaVersion: number; + createStatements: string[]; + migrations: { version: number; statements: string[] }[]; + query(sql: string, parameters: ClientSQLDebugValue[] | undefined): Promise; + debugInfo?(): Promise>; +} + +interface ClientSQLDebugTableSummary { + name: string; + type: string; + sql: string | null; + rowCount: number | null; + rowCountIsLowerBound: boolean; + columns: ClientSQLDebugColumn[]; + truncation: { + columns: boolean; + omittedColumnsAtLeast: number; + rowCount: boolean; + }; +} + +interface ClientSQLDebugTruncationMetadata { + truncated: boolean; + reasons: string[]; + limits: { + databases: number; + tables: number; + columnsPerTable: number; + schemaColumns: number; + rows: number; + cells: number; + collectionItems: number; + stringCharacters: number; + blobBytes: number; + payloadBytes: number; + }; + omittedDatabases: number; + omittedTables: number; + omittedColumns: number; + omittedRows: number; + omittedCells: number; + omittedValues: number; + truncatedStrings: number; + truncatedBlobs: number; + payloadBytes: number; +} + +const CLIENT_SQL_DEBUG_SCHEMA_CACHE_TTL_MS = 2000; +const CLIENT_SQL_DEBUG_MAX_DATABASES = 8; +const CLIENT_SQL_DEBUG_MAX_TABLES = 32; +const CLIENT_SQL_DEBUG_MAX_COLUMNS_PER_TABLE = 50; +const CLIENT_SQL_DEBUG_MAX_SCHEMA_COLUMNS = 512; +const CLIENT_SQL_DEBUG_DEFAULT_ROWS = 100; +const CLIENT_SQL_DEBUG_MAX_ROWS = 100; +const CLIENT_SQL_DEBUG_MAX_OFFSET = 1000000; +const CLIENT_SQL_DEBUG_MAX_CELLS = 5000; +const CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS = 4096; +const CLIENT_SQL_DEBUG_MAX_BLOB_BYTES = 4096; +// Keep adapter documents comfortably below the generic provider's 48 KiB action-document cap. +const CLIENT_SQL_DEBUG_MAX_PAYLOAD_BYTES = 40 * 1024; +// Reserve space for the fixed truncation envelope inside the adapter document cap. +const CLIENT_SQL_DEBUG_TRUNCATION_METADATA_BYTES = 2048; +const CLIENT_SQL_DEBUG_MAX_CONTENT_BYTES = + CLIENT_SQL_DEBUG_MAX_PAYLOAD_BYTES - CLIENT_SQL_DEBUG_TRUNCATION_METADATA_BYTES; +const CLIENT_SQL_DEBUG_MAX_QUERY_VALUE_BYTES = 8 * 1024 * 1024; +const CLIENT_SQL_DEBUG_MAX_VALUES = 8192; +// Match the generic provider parser: it rejects collection item 101. +const CLIENT_SQL_DEBUG_MAX_COLLECTION_ITEMS = 100; +const CLIENT_SQL_DEBUG_MAX_VALUE_DEPTH = 8; +const CLIENT_SQL_DEBUG_MAX_ROW_COUNT_SCAN = 10000; +const CLIENT_SQL_DEBUG_ESTIMATED_UTF8_BYTES_PER_CHARACTER = 4; +const CLIENT_SQL_DEBUG_PROVIDER_OWNER_KEY = 'client_sql/src/ClientSQLDebug'; +const databasesById: { [id: string]: ClientSQLDebugDatabase[] | undefined } = Object.create(null); +const databaseSummaryCacheById: { + [id: string]: { cachedAt: number; summary: Record } | undefined; +} = Object.create(null); +let debugProviderRegistration: DebuggerProviderRegistration | undefined; +let debugProviderOwner: DebuggerProviderOwner | undefined; +let debugRevision = 0; + +class ClientSQLDebugPayloadLimiter { + private reasons: string[] = []; + private bytesUsed = 0; + private valueCount = 0; + private omittedDatabases = 0; + private omittedTables = 0; + private omittedColumns = 0; + private omittedRows = 0; + private omittedCells = 0; + private omittedValues = 0; + private truncatedStrings = 0; + private truncatedBlobs = 0; + + absorbMetadata(metadata: unknown): void { + if (!metadata || typeof metadata !== 'object') { + return; + } + const source = metadata as Partial; + if (Array.isArray(source.reasons)) { + source.reasons.forEach(reason => { + if (typeof reason === 'string') { + this.noteReason(reason); + } + }); + } + this.omittedDatabases += this.metadataCount(source.omittedDatabases); + this.omittedTables += this.metadataCount(source.omittedTables); + this.omittedColumns += this.metadataCount(source.omittedColumns); + this.omittedRows += this.metadataCount(source.omittedRows); + this.omittedCells += this.metadataCount(source.omittedCells); + this.omittedValues += this.metadataCount(source.omittedValues); + this.truncatedStrings += this.metadataCount(source.truncatedStrings); + this.truncatedBlobs += this.metadataCount(source.truncatedBlobs); + } + + noteReason(reason: string): void { + if (this.reasons.indexOf(reason) === -1) { + this.reasons.push(reason); + } + } + + noteOmittedDatabases(count: number): void { + if (count > 0) { + this.omittedDatabases += count; + this.noteReason('databases'); + } + } + + noteOmittedTables(count: number): void { + if (count > 0) { + this.omittedTables += count; + this.noteReason('tables'); + } + } + + noteOmittedColumns(count: number): void { + if (count > 0) { + this.omittedColumns += count; + this.noteReason('columns'); + } + } + + noteOmittedRows(count: number): void { + if (count > 0) { + this.omittedRows += count; + this.noteReason('rows'); + } + } + + noteOmittedCells(count: number): void { + if (count > 0) { + this.omittedCells += count; + this.noteReason('cells'); + } + } + + noteTruncatedString(): void { + this.truncatedStrings += 1; + this.noteReason('stringValues'); + } + + noteTruncatedBlob(): void { + this.truncatedBlobs += 1; + this.noteReason('blobValues'); + } + + limitValue(value: unknown, depth: number): unknown { + if (depth > CLIENT_SQL_DEBUG_MAX_VALUE_DEPTH) { + this.omittedValues += 1; + this.noteReason('valueDepth'); + return null; + } + if (this.valueCount >= CLIENT_SQL_DEBUG_MAX_VALUES) { + this.omittedValues += 1; + this.noteReason('values'); + return null; + } + this.valueCount += 1; + + if (value === null || value === undefined) { + this.reservePrimitive(); + return null; + } + if (typeof value === 'string') { + return this.limitString(value); + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + this.noteReason('nonFiniteValues'); + return this.limitString(String(value)); + } + return this.reservePrimitive() ? value : null; + } + if (typeof value === 'boolean') { + return this.reservePrimitive() ? value : null; + } + if (value instanceof ArrayBuffer) { + return this.limitBlob(value); + } + if (Array.isArray(value)) { + if (!this.reserveBytes(8)) { + return []; + } + const values = value.slice(0, CLIENT_SQL_DEBUG_MAX_COLLECTION_ITEMS); + if (values.length < value.length) { + this.omittedValues += value.length - values.length; + this.noteReason('collectionValues'); + } + const output: unknown[] = []; + for (let index = 0; index < values.length; index += 1) { + if (this.isExhausted()) { + this.omittedValues += values.length - index; + this.noteReason('payloadBytes'); + break; + } + output.push(this.limitValue(values[index], depth + 1)); + } + return output; + } + if (typeof value === 'object') { + if (!this.reserveBytes(16)) { + return {}; + } + const input = value as Record; + const keys = Object.keys(input); + const selectedKeys = keys.slice(0, CLIENT_SQL_DEBUG_MAX_COLLECTION_ITEMS); + if (selectedKeys.length < keys.length) { + this.omittedValues += keys.length - selectedKeys.length; + this.noteReason('objectProperties'); + } + const output: Record = {}; + for (let index = 0; index < selectedKeys.length; index += 1) { + if (this.isExhausted()) { + this.omittedValues += selectedKeys.length - index; + this.noteReason('payloadBytes'); + break; + } + const key = selectedKeys[index]; + let limitedKey = this.limitString(key); + if (Object.prototype.hasOwnProperty.call(output, limitedKey)) { + limitedKey = `${limitedKey.slice(0, CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS - 16)}#${index}`; + this.noteReason('propertyNames'); + } + output[limitedKey] = this.limitValue(input[key], depth + 1); + } + return output; + } + return this.limitString(String(value)); + } + + metadata(payloadBytes: number): ClientSQLDebugTruncationMetadata { + return { + truncated: this.reasons.length > 0, + reasons: this.reasons.slice(), + limits: { + databases: CLIENT_SQL_DEBUG_MAX_DATABASES, + tables: CLIENT_SQL_DEBUG_MAX_TABLES, + columnsPerTable: CLIENT_SQL_DEBUG_MAX_COLUMNS_PER_TABLE, + schemaColumns: CLIENT_SQL_DEBUG_MAX_SCHEMA_COLUMNS, + rows: CLIENT_SQL_DEBUG_MAX_ROWS, + cells: CLIENT_SQL_DEBUG_MAX_CELLS, + collectionItems: CLIENT_SQL_DEBUG_MAX_COLLECTION_ITEMS, + stringCharacters: CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS, + blobBytes: CLIENT_SQL_DEBUG_MAX_BLOB_BYTES, + payloadBytes: CLIENT_SQL_DEBUG_MAX_PAYLOAD_BYTES, + }, + omittedDatabases: this.omittedDatabases, + omittedTables: this.omittedTables, + omittedColumns: this.omittedColumns, + omittedRows: this.omittedRows, + omittedCells: this.omittedCells, + omittedValues: this.omittedValues, + truncatedStrings: this.truncatedStrings, + truncatedBlobs: this.truncatedBlobs, + payloadBytes, + }; + } + + private reservePrimitive(): boolean { + if (!this.reserveBytes(8)) { + this.omittedValues += 1; + return false; + } + return true; + } + + private metadataCount(value: unknown): number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : 0; + } + + private isExhausted(): boolean { + return this.bytesUsed + 8 > CLIENT_SQL_DEBUG_MAX_CONTENT_BYTES + || this.valueCount >= CLIENT_SQL_DEBUG_MAX_VALUES; + } + + private reserveBytes(bytes: number): boolean { + if (this.bytesUsed + bytes > CLIENT_SQL_DEBUG_MAX_CONTENT_BYTES) { + this.noteReason('payloadBytes'); + return false; + } + this.bytesUsed += bytes; + return true; + } + + private limitString(value: string): string { + const remainingBytes = Math.max(0, CLIENT_SQL_DEBUG_MAX_CONTENT_BYTES - this.bytesUsed - 8); + const payloadCharacters = Math.floor(remainingBytes / 2); + const characterCount = Math.min( + value.length, + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS, + payloadCharacters, + ); + const limited = value.slice(0, characterCount); + if (limited.length < value.length) { + this.noteTruncatedString(); + if (payloadCharacters < Math.min(value.length, CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS)) { + this.noteReason('payloadBytes'); + } + } + this.reserveBytes(limited.length * 2 + 8); + return limited; + } + + private limitBlob(value: ArrayBuffer): Record { + const remainingBytes = Math.max(0, CLIENT_SQL_DEBUG_MAX_CONTENT_BYTES - this.bytesUsed - 8); + const maximumBytesForHex = Math.floor(Math.max(0, remainingBytes - 128) / 2); + const byteCount = Math.min(value.byteLength, CLIENT_SQL_DEBUG_MAX_BLOB_BYTES, maximumBytesForHex); + const bytes = new Uint8Array(value, 0, byteCount); + let data = ''; + for (let index = 0; index < bytes.length; index += 1) { + data += bytes[index].toString(16).padStart(2, '0'); + } + if (byteCount < value.byteLength) { + this.noteTruncatedBlob(); + if (maximumBytesForHex < Math.min(value.byteLength, CLIENT_SQL_DEBUG_MAX_BLOB_BYTES)) { + this.noteReason('payloadBytes'); + } + } + this.reserveBytes(data.length + 128); + return { + type: 'blob', + encoding: 'hex', + byteLength: value.byteLength, + shownBytes: byteCount, + truncated: byteCount < value.byteLength, + data, + }; + } +} + +function limitDebugPayload( + value: Record, + limiter: ClientSQLDebugPayloadLimiter, +): Record { + return limiter.limitValue(value, 0) as Record; +} + +function utf8ByteLength(value: string): number { + let bytes = 0; + for (let index = 0; index < value.length; index += 1) { + const codeUnit = value.charCodeAt(index); + if (codeUnit <= 0x7f) { + bytes += 1; + } else if (codeUnit <= 0x7ff) { + bytes += 2; + } else if ( + codeUnit >= 0xd800 && codeUnit <= 0xdbff + && index + 1 < value.length + && value.charCodeAt(index + 1) >= 0xdc00 + && value.charCodeAt(index + 1) <= 0xdfff + ) { + bytes += 4; + index += 1; + } else { + bytes += 3; + } + } + return bytes; +} + +function jsonPayloadBytes(value: unknown): number { + return utf8ByteLength(JSON.stringify(value) ?? 'null'); +} + +function assignExactPayloadBytes( + value: Record, + limiter: ClientSQLDebugPayloadLimiter, +): number { + let previousBytes = -1; + for (let attempt = 0; attempt < 8; attempt += 1) { + const payloadBytes = jsonPayloadBytes(value); + value.truncation = limiter.metadata(payloadBytes); + if (payloadBytes === previousBytes) { + return payloadBytes; + } + previousBytes = payloadBytes; + } + const payloadBytes = jsonPayloadBytes(value); + value.truncation = limiter.metadata(payloadBytes); + return jsonPayloadBytes(value); +} + +function serializeDebugProviderDocument(value: Record): string { + const limiter = new ClientSQLDebugPayloadLimiter(); + limiter.absorbMetadata(value.truncation); + const limited = limitDebugPayload(value, limiter); + limited.truncation = limiter.metadata(0); + let payloadBytes = assignExactPayloadBytes(limited, limiter); + if (payloadBytes <= CLIENT_SQL_DEBUG_MAX_PAYLOAD_BYTES) { + const document = JSON.stringify(limited); + if (utf8ByteLength(document) > CLIENT_SQL_DEBUG_MAX_PAYLOAD_BYTES) { + throw new Error('ClientSQL debugger serialized document exceeded its payload byte cap.'); + } + return document; + } + + limiter.noteReason('payloadBytes'); + limiter.noteReason('finalByteCap'); + const fallback: Record = { + source: typeof limited.source === 'string' ? limited.source.slice(0, 128) : 'target', + revision: typeof limited.revision === 'number' ? limited.revision : debugRevision, + truncation: limiter.metadata(0), + }; + for (const field of ['databaseId', 'databaseName', 'table']) { + const fieldValue = limited[field]; + if (typeof fieldValue === 'string') { + fallback[field] = fieldValue.slice(0, 128); + } + } + payloadBytes = assignExactPayloadBytes(fallback, limiter); + if (payloadBytes > CLIENT_SQL_DEBUG_MAX_PAYLOAD_BYTES) { + throw new Error('ClientSQL debugger could not fit truncation metadata within its payload byte cap.'); + } + return JSON.stringify(fallback); +} + +function createClientSQLDebuggerProviderResult( + value: Record, +): DebuggerProviderResult { + return createDebuggerProviderResult(serializeDebugProviderDocument(value)); +} + +function nextDebugRevision(): number { + debugRevision += 1; + debugProviderRegistration?.notifyChange(); + return debugRevision; +} + +function requestInteger(data: any, field: string, fallback: number): number { + const value = data?.[field]; + if (value === undefined) { + return fallback; + } + if (typeof value !== 'number' || !Number.isFinite(value) || !Number.isSafeInteger(value)) { + throw new Error(`ClientSQL debugger '${field}' must be a finite integer.`); + } + return value; +} + +function clampInteger(value: number, minimum: number, maximum: number): number { + return Math.min(maximum, Math.max(minimum, value)); +} + +function quoteSQLiteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; +} + +function clientSQLRowsOrderClause(tableType: string | null | undefined, columns: ClientSQLDebugColumn[]): string { + const primaryKeyColumns = columns + .filter(column => column.pk > 0) + .sort((a, b) => a.pk - b.pk); + if (primaryKeyColumns.length) { + return ` ORDER BY ${primaryKeyColumns.map(column => `${quoteSQLiteIdentifier(column.name)} DESC`).join(', ')}`; + } + if (tableType === 'table') { + return ' ORDER BY rowid DESC'; + } + return ''; +} + +function firstOpenDatabaseById(): ClientSQLDebugDatabase[] { + return Object.keys(databasesById) + .map(id => databasesById[id]?.[0]) + .filter((database): database is ClientSQLDebugDatabase => database !== undefined); +} + +function clearDatabaseSummaryCache(id: string): void { + delete databaseSummaryCacheById[id]; +} + +function debugErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function summarizeTable( + database: ClientSQLDebugDatabase, + table: { name: string; type: string; sql: string | null }, + columnLimit: number, +): Promise { + const overlongTableName = table.name.length > CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS; + const safeTableName = table.name.slice(0, CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS); + if (overlongTableName || columnLimit <= 0) { + return { + name: safeTableName, + type: table.type, + sql: table.sql, + rowCount: null, + rowCountIsLowerBound: false, + columns: [], + truncation: { + columns: true, + omittedColumnsAtLeast: 1, + rowCount: true, + }, + }; + } + + const appliedColumnLimit = Math.min(CLIENT_SQL_DEBUG_MAX_COLUMNS_PER_TABLE, columnLimit); + const columns = await database.query( + 'SELECT cid, substr(name, 1, ?) AS name, substr(type, 1, ?) AS type, ' + + '"notnull" AS "notnull", ' + + 'CASE WHEN dflt_value IS NULL THEN NULL ELSE substr(CAST(dflt_value AS TEXT), 1, ?) END AS dflt_value, pk ' + + 'FROM pragma_table_info(?) ORDER BY cid LIMIT ?', + [ + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, + table.name, + appliedColumnLimit + 1, + ], + ); + const safeColumns = columns + .slice(0, appliedColumnLimit) + .filter(column => column.name.length <= CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS); + const omittedColumnsAtLeast = Math.max(0, columns.length - safeColumns.length); + let rowCount: number | null = null; + let rowCountIsLowerBound = false; + let rowCountUnavailable = false; + try { + const countRows = await database.query<{ count: number }>( + `SELECT COUNT(*) AS count FROM (` + + `SELECT 1 FROM ${quoteSQLiteIdentifier(table.name)} LIMIT ?` + + ')', + [CLIENT_SQL_DEBUG_MAX_ROW_COUNT_SCAN + 1], + ); + const count = Number(countRows[0]?.count ?? 0); + if (!Number.isFinite(count) || !Number.isSafeInteger(count) || count < 0) { + throw new Error(`received invalid count '${String(countRows[0]?.count)}'`); + } + rowCountIsLowerBound = count > CLIENT_SQL_DEBUG_MAX_ROW_COUNT_SCAN; + rowCount = rowCountIsLowerBound ? CLIENT_SQL_DEBUG_MAX_ROW_COUNT_SCAN : count; + } catch (error) { + rowCountUnavailable = true; + console.warn( + `ClientSQL debugger could not count rows for database '${database.name}', ` + + `table '${safeTableName}': ${debugErrorMessage(error)}`, + ); + } + + return { + name: safeTableName, + type: table.type, + sql: table.sql, + rowCount, + rowCountIsLowerBound, + columns: safeColumns, + truncation: { + columns: omittedColumnsAtLeast > 0, + omittedColumnsAtLeast, + rowCount: rowCountIsLowerBound || rowCountUnavailable, + }, + }; +} + +async function summarizeDatabase(database: ClientSQLDebugDatabase): Promise> { + const limiter = new ClientSQLDebugPayloadLimiter(); + const versionRows = await database.query<{ user_version: number }>('PRAGMA user_version', []); + const tables = await database.query<{ name: string; type: string; sql: string | null }>( + "SELECT substr(name, 1, ?) AS name, type, " + + "CASE WHEN sql IS NULL THEN NULL ELSE substr(sql, 1, ?) END AS sql " + + "FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' " + + 'ORDER BY type, name LIMIT ?', + [ + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, + CLIENT_SQL_DEBUG_MAX_TABLES + 1, + ], + ); + const selectedTables = tables.slice(0, CLIENT_SQL_DEBUG_MAX_TABLES); + limiter.noteOmittedTables(Math.max(0, tables.length - selectedTables.length)); + const tableSummaries: ClientSQLDebugTableSummary[] = []; + let remainingColumns = CLIENT_SQL_DEBUG_MAX_SCHEMA_COLUMNS; + for (const table of selectedTables) { + const summary = await summarizeTable(database, table, remainingColumns); + tableSummaries.push(summary); + remainingColumns -= summary.columns.length; + limiter.noteOmittedColumns(summary.truncation.omittedColumnsAtLeast); + if (summary.truncation.rowCount) { + limiter.noteReason('rowCount'); + } + } + const debugInfo = database.debugInfo ? await database.debugInfo() : null; + + const rawSummary = { + id: `target:${database.id}`, + name: database.name, + path: `${database.name} (target ClientSQL)`, + root: 'debugger-target', + size: null, + modifiedAt: null, + userVersion: Number(versionRows[0]?.user_version ?? database.schemaVersion), + debugInfo, + tables: tableSummaries, + }; + const limitedSummary = limitDebugPayload(rawSummary, limiter); + const limitedTables = Array.isArray(limitedSummary.tables) + ? limitedSummary.tables as Array> + : []; + limitedSummary.tables = limitedTables; + limiter.noteOmittedTables(Math.max(0, tableSummaries.length - limitedTables.length)); + const rawColumnCount = tableSummaries.reduce((count, table) => count + table.columns.length, 0); + const limitedColumnCount = limitedTables.reduce((count, table) => { + return count + (Array.isArray(table.columns) ? table.columns.length : 0); + }, 0); + limiter.noteOmittedColumns(Math.max(0, rawColumnCount - limitedColumnCount)); + limitedSummary.truncation = limiter.metadata(0); + return limitedSummary; +} + +async function summarizeDatabaseCached( + database: ClientSQLDebugDatabase, + forceRefresh: boolean, +): Promise> { + const cached = databaseSummaryCacheById[database.id]; + if (!forceRefresh && cached && Date.now() - cached.cachedAt < CLIENT_SQL_DEBUG_SCHEMA_CACHE_TTL_MS) { + return cached.summary; + } + + const summary = await summarizeDatabase(database); + databaseSummaryCacheById[database.id] = { + cachedAt: Date.now(), + summary, + }; + return summary; +} + +async function summarizeSingleTable( + database: ClientSQLDebugDatabase, + tableName: string, +): Promise { + if (tableName.length > CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS) { + throw new Error(`ClientSQL table name exceeds ${CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS} characters.`); + } + const tables = await database.query<{ name: string; type: string; sql: string | null }>( + "SELECT substr(name, 1, ?) AS name, type, " + + "CASE WHEN sql IS NULL THEN NULL ELSE substr(sql, 1, ?) END AS sql " + + "FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' " + + 'AND name = ? LIMIT 1', + [CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS + 1, tableName], + ); + const table = tables[0]; + if (!table) { + throw new Error(`Unknown ClientSQL table '${tableName}'.`); + } + return summarizeTable(database, table, CLIENT_SQL_DEBUG_MAX_COLUMNS_PER_TABLE); +} + +async function inspectDatabases(data: any): Promise> { + const forceRefresh = Boolean(data?.refresh); + const limiter = new ClientSQLDebugPayloadLimiter(); + const databases = firstOpenDatabaseById(); + const selectedDatabases = databases.slice(0, CLIENT_SQL_DEBUG_MAX_DATABASES); + limiter.noteOmittedDatabases(Math.max(0, databases.length - selectedDatabases.length)); + const summaries = await Promise.all( + selectedDatabases.map(database => summarizeDatabaseCached(database, forceRefresh)), + ); + summaries.forEach(summary => limiter.absorbMetadata(summary.truncation)); + const rawSummary = { + source: 'target', + revision: debugRevision, + roots: ['debugger-target'], + databases: summaries, + }; + const limitedSummary = limitDebugPayload(rawSummary, limiter); + const limitedDatabases = Array.isArray(limitedSummary.databases) ? limitedSummary.databases : []; + limitedSummary.databases = limitedDatabases; + limiter.noteOmittedDatabases(Math.max(0, summaries.length - limitedDatabases.length)); + limitedSummary.truncation = limiter.metadata(0); + return limitedSummary; +} + +function rowSelectExpression(column: ClientSQLDebugColumn, index: number): string { + const identifier = quoteSQLiteIdentifier(column.name); + return `CASE WHEN typeof(${identifier}) = 'text' THEN ` + + `substr(${identifier}, 1, ${CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS}) ` + + `WHEN typeof(${identifier}) = 'blob' THEN ` + + `substr(${identifier}, 1, ${CLIENT_SQL_DEBUG_MAX_BLOB_BYTES}) ` + + `ELSE ${identifier} END AS ${quoteSQLiteIdentifier(`__clientsql_value_${index}`)}, ` + + `typeof(${identifier}) AS ${quoteSQLiteIdentifier(`__clientsql_type_${index}`)}, ` + + `CASE WHEN typeof(${identifier}) IN ('text', 'blob') THEN length(${identifier}) ELSE 0 END ` + + `AS ${quoteSQLiteIdentifier(`__clientsql_length_${index}`)}`; +} + +async function inspectTable(data: any): Promise> { + const databaseId = String(data?.databaseId ?? ''); + const identity = databaseId.startsWith('target:') ? databaseId.slice('target:'.length) : databaseId; + const database = databasesById[identity]?.[0]; + if (!database) { + throw new Error(`Unknown ClientSQL database '${identity}'.`); + } + + const table = String(data?.table ?? ''); + if (!table) { + throw new Error('Missing required table.'); + } + + const requestedLimit = requestInteger(data, 'limit', CLIENT_SQL_DEBUG_DEFAULT_ROWS); + const requestedOffset = requestInteger(data, 'offset', 0); + const validLimit = clampInteger(requestedLimit, 1, CLIENT_SQL_DEBUG_MAX_ROWS); + const offset = clampInteger(requestedOffset, 0, CLIENT_SQL_DEBUG_MAX_OFFSET); + const tableSummary = await summarizeSingleTable(database, table); + const columns = tableSummary.columns.slice(0, CLIENT_SQL_DEBUG_MAX_COLUMNS_PER_TABLE); + if (!columns.length) { + throw new Error(`ClientSQL table '${table}' has no inspectable columns.`); + } + + const maximumCellBytes = Math.max( + CLIENT_SQL_DEBUG_MAX_BLOB_BYTES, + CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS * CLIENT_SQL_DEBUG_ESTIMATED_UTF8_BYTES_PER_CHARACTER, + ); + const maximumRowsForQueryWork = Math.max( + 1, + Math.floor(CLIENT_SQL_DEBUG_MAX_QUERY_VALUE_BYTES / (columns.length * maximumCellBytes)), + ); + const limit = Math.min(validLimit, maximumRowsForQueryWork); + const orderClause = clientSQLRowsOrderClause(tableSummary.type, columns); + const selectExpressions = columns.map(rowSelectExpression).join(', '); + const queriedRows = await database.query>( + `SELECT ${selectExpressions} FROM ${quoteSQLiteIdentifier(table)}` + + `${orderClause} LIMIT ? OFFSET ?`, + [limit + 1, offset], + ); + const selectedRows = queriedRows.slice(0, limit); + // Rows are positional arrays aligned with the separately bounded `columns` + // metadata. SQL identifiers are valid far beyond the generic provider's + // JSON property-name limit, so they must never become bridge object keys. + const rows = selectedRows.map(row => columns.map((_column, index) => { + return row[`__clientsql_value_${index}`]; + })); + + const limiter = new ClientSQLDebugPayloadLimiter(); + if (requestedLimit !== validLimit) { + limiter.noteOmittedRows(Math.max(0, requestedLimit - validLimit)); + limiter.noteReason('requestedRowLimit'); + } + if (validLimit > limit) { + limiter.noteOmittedRows(validLimit - limit); + limiter.noteReason('queryWork'); + } + if (queriedRows.length > selectedRows.length) { + limiter.noteOmittedRows(queriedRows.length - selectedRows.length); + } + if (requestedOffset !== offset) { + limiter.noteReason('requestedOffset'); + } + limiter.noteOmittedColumns(tableSummary.truncation.omittedColumnsAtLeast); + if (tableSummary.truncation.rowCount) { + limiter.noteReason('rowCount'); + } + const selectedCellCount = rows.length * columns.length; + if (selectedCellCount > CLIENT_SQL_DEBUG_MAX_CELLS) { + limiter.noteOmittedCells(selectedCellCount - CLIENT_SQL_DEBUG_MAX_CELLS); + } + selectedRows.forEach(row => { + columns.forEach((column, index) => { + const valueType = String(row[`__clientsql_type_${index}`]); + const rawLength = Number(row[`__clientsql_length_${index}`] ?? 0); + if (valueType === 'text' && rawLength > CLIENT_SQL_DEBUG_MAX_STRING_CHARACTERS) { + limiter.noteTruncatedString(); + } + if (valueType === 'blob' && rawLength > CLIENT_SQL_DEBUG_MAX_BLOB_BYTES) { + limiter.noteTruncatedBlob(); + } + }); + }); + + const rawSummary = { + source: 'target', + revision: debugRevision, + databaseId: `target:${database.id}`, + databaseName: database.name, + table, + requestedLimit, + limit, + requestedOffset, + offset, + rowCount: tableSummary.rowCount, + rowCountIsLowerBound: tableSummary.rowCountIsLowerBound, + hasMore: queriedRows.length > selectedRows.length, + orderBy: orderClause.replace(/^\s*ORDER BY\s+/i, '') || null, + columns, + rows, + }; + const limitedSummary = limitDebugPayload(rawSummary, limiter); + const limitedColumns = Array.isArray(limitedSummary.columns) ? limitedSummary.columns : []; + const limitedRows = Array.isArray(limitedSummary.rows) + ? (limitedSummary.rows as unknown[][]).map(row => row.slice(0, limitedColumns.length)) + : []; + limitedSummary.columns = limitedColumns; + limitedSummary.rows = limitedRows; + limiter.noteOmittedColumns(Math.max(0, columns.length - limitedColumns.length)); + limiter.noteOmittedRows(Math.max(0, rows.length - limitedRows.length)); + const limitedCellCount = limitedRows.reduce((count, row) => count + row.length, 0); + limiter.noteOmittedCells(Math.max(0, selectedCellCount - limitedCellCount)); + limitedSummary.truncation = limiter.metadata(0); + return limitedSummary; +} + +function ensureDebugProviderRegistered(): void { + if (debugProviderRegistration) { + return; + } + if (!debugProviderOwner) { + debugProviderOwner = createDebuggerProviderOwner(module, CLIENT_SQL_DEBUG_PROVIDER_OWNER_KEY); + } + debugProviderRegistration = debugProviderOwner.register({ + availability: () => { + const databaseCount = firstOpenDatabaseById().length; + return { + available: databaseCount > 0, + message: databaseCount > 0 + ? `${databaseCount} open ClientSQL database${databaseCount === 1 ? '' : 's'}.` + : 'No ClientSQL databases are currently open.', + }; + }, + description: 'Inspect open ClientSQL databases through bounded, read-only queries.', + handleRequest: async (request: DebuggerProviderRequest): Promise => { + if (request.action === 'list') { + return createClientSQLDebuggerProviderResult(await inspectDatabases(request)); + } + if (request.action === 'table') { + return createClientSQLDebuggerProviderResult(await inspectTable(request)); + } + throw new Error(`Unsupported ClientSQL debugger action: ${request.action}`); + }, + id: 'client-sql', + kind: DebuggerProviderKind.Sql, + label: 'ClientSQL', + }); +} + +function releaseDebugProviderIfUnused(): void { + if (Object.keys(databasesById).length > 0 || !debugProviderRegistration) { + return; + } + debugProviderRegistration.dispose(); + debugProviderRegistration = undefined; +} + +export function registerClientSQLDebugDatabase(database: ClientSQLDebugDatabase): void { + ensureDebugProviderRegistered(); + let databases = databasesById[database.id]; + if (!databases) { + databases = []; + databasesById[database.id] = databases; + } + if (databases.indexOf(database) === -1) { + databases.push(database); + } + clearDatabaseSummaryCache(database.id); + nextDebugRevision(); +} + +export function unregisterClientSQLDebugDatabase(database: ClientSQLDebugDatabase): void { + const databases = databasesById[database.id]; + if (!databases) { + return; + } + + const index = databases.indexOf(database); + if (index !== -1) { + databases.splice(index, 1); + } + if (!databases.length) { + delete databasesById[database.id]; + } + clearDatabaseSummaryCache(database.id); + nextDebugRevision(); + releaseDebugProviderIfUnused(); +} + +export function notifyClientSQLDebugChanged(databaseId: string | undefined): void { + if (databaseId) { + clearDatabaseSummaryCache(databaseId); + } else { + Object.keys(databaseSummaryCacheById).forEach(clearDatabaseSummaryCache); + } + nextDebugRevision(); +} diff --git a/src/valdi_modules/src/valdi/client_sql/src/ClientSQLNative.d.ts b/src/valdi_modules/src/valdi/client_sql/src/ClientSQLNative.d.ts new file mode 100644 index 000000000..107f6df2f --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/src/ClientSQLNative.d.ts @@ -0,0 +1,112 @@ +/** + * @ExportModule + * @Version(__PLACEHOLDER__) + */ + +export type ClientSQLValue = string | number | boolean | ArrayBuffer | null; + +export type ClientSQLNativeCallback = (value: T | undefined, error: string | undefined) => void; + +/** + * SQL parameters and result rows are heterogeneous by statement, so this payload carrier cannot + * have one stable typed native schema. Only this value boundary intentionally marshals as untyped; + * the exported module, migration model, and connection proxies remain typed native contracts. + * + * @NativeClass({ + * marshallAsUntyped: true, + * ios: 'NSObject', iosImportPrefix: 'Foundation', + * android: 'kotlin.Any' + * }) + */ +export interface ClientSQLNativeAny {} + +/** + * @ExportModel + * @Version(__PLACEHOLDER__) + */ +export interface ClientSQLMigration { + version: number; + statements: string[]; +} + +/** + * @ExportProxy + * @Version(__PLACEHOLDER__) + */ +export interface ClientSQLNativeTransactionProxy { + execute( + sql: string, + parameters: ClientSQLNativeAny[] | undefined, + callback: (value: ClientSQLNativeAny | undefined, error: string | undefined) => void, + ): void; + query( + sql: string, + parameters: ClientSQLNativeAny[] | undefined, + callback: (value: ClientSQLNativeAny[] | undefined, error: string | undefined) => void, + ): void; +} + +/** + * @ExportProxy + * @Version(__PLACEHOLDER__) + */ +export interface ClientSQLNativeConnectionProxy { + execute( + sql: string, + parameters: ClientSQLNativeAny[] | undefined, + callback: (value: ClientSQLNativeAny | undefined, error: string | undefined) => void, + ): void; + query( + sql: string, + parameters: ClientSQLNativeAny[] | undefined, + callback: (value: ClientSQLNativeAny[] | undefined, error: string | undefined) => void, + ): void; + queryOnWriter( + sql: string, + parameters: ClientSQLNativeAny[] | undefined, + callback: (value: ClientSQLNativeAny[] | undefined, error: string | undefined) => void, + ): void; + transaction( + body: ( + transaction: ClientSQLNativeTransactionProxy, + callback: (value: ClientSQLNativeAny | undefined, error: string | undefined) => void, + ) => void, + callback: (value: ClientSQLNativeAny | undefined, error: string | undefined) => void, + ): void; + debugInfo(callback: (value: ClientSQLNativeAny | undefined, error: string | undefined) => void): void; + close(callback: (value: ClientSQLNativeAny | undefined, error: string | undefined) => void): void; +} + +export interface ClientSQLNativeTransaction { + execute(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void; + query(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void; +} + +export type ClientSQLNativeTransactionBody = ( + transaction: ClientSQLNativeTransaction, + callback: ClientSQLNativeCallback, +) => void; + +export interface ClientSQLNativeConnection { + execute(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void; + query(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void; + transaction(body: ClientSQLNativeTransactionBody, callback: ClientSQLNativeCallback): void; + close(callback: ClientSQLNativeCallback): void; +} + +export interface ClientSQLNativeModule { + /** `name` is the generator's canonical module-scoped storage identity, not a filesystem path. */ + openDatabase( + name: string, + schemaVersion: number, + createStatements: string[], + migrations: ClientSQLMigration[], + ): ClientSQLNativeConnection; +} + +export function openDatabase( + name: string, + schemaVersion: number, + createStatements: string[], + migrations: ClientSQLMigration[], +): ClientSQLNativeConnectionProxy; diff --git a/src/valdi_modules/src/valdi/client_sql/test/ClientSQLDebug.spec.ts b/src/valdi_modules/src/valdi/client_sql/test/ClientSQLDebug.spec.ts new file mode 100644 index 000000000..2aa84ea55 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/test/ClientSQLDebug.spec.ts @@ -0,0 +1,219 @@ +import 'jasmine/src/jasmine'; +import * as DebuggerProvider from 'valdi_core/src/debugging/DebuggerProvider'; +import type { + DebuggerProvider as DebuggerProviderContract, + DebuggerProviderModule, + DebuggerProviderOwner, + DebuggerProviderRegistration, +} from 'valdi_core/src/debugging/DebuggerProvider'; +import { + ClientSQLDebugDatabase, + ClientSQLDebugValue, + notifyClientSQLDebugChanged, + registerClientSQLDebugDatabase, + unregisterClientSQLDebugDatabase, +} from '../src/ClientSQLDebug'; + +const LONG_COLUMN_NAME = `long_${'x'.repeat(1500)}`; +const PENDING_CHANGED_TABLES = Array.from( + { length: 101 }, + (_value, index) => `pending_table_${index.toString().padStart(3, '0')}`, +); + +function utf8ByteLength(value: string): number { + let bytes = 0; + for (let index = 0; index < value.length; index += 1) { + const codeUnit = value.charCodeAt(index); + if (codeUnit <= 0x7f) { + bytes += 1; + } else if (codeUnit <= 0x7ff) { + bytes += 2; + } else if ( + codeUnit >= 0xd800 && codeUnit <= 0xdbff + && index + 1 < value.length + && value.charCodeAt(index + 1) >= 0xdc00 + && value.charCodeAt(index + 1) <= 0xdfff + ) { + bytes += 4; + index += 1; + } else { + bytes += 3; + } + } + return bytes; +} + +function makeDebugDatabase(id: string): ClientSQLDebugDatabase { + const blob = new Uint8Array(8192); + blob.fill(0xab); + const debugRow: Record = { + __clientsql_value_0: '\u0000\u0001'.repeat(4096), + __clientsql_type_0: 'text', + __clientsql_length_0: 8192, + __clientsql_value_1: '😀'.repeat(4096), + __clientsql_type_1: 'text', + __clientsql_length_1: 8192, + __clientsql_value_2: blob.buffer, + __clientsql_type_2: 'blob', + __clientsql_length_2: blob.byteLength, + }; + return { + id, + name: 'Debug database', + schemaVersion: 1, + createStatements: [], + migrations: [], + debugInfo(): Promise> { + return Promise.resolve({ + pendingChangedTables: PENDING_CHANGED_TABLES.slice(), + }); + }, + query(sql: string, _parameters: ClientSQLDebugValue[] | undefined): Promise { + if (sql === 'PRAGMA user_version') { + return Promise.resolve([{ user_version: 1 }] as unknown as T[]); + } + if (sql.indexOf('FROM sqlite_schema') !== -1) { + return Promise.resolve([{ + name: 'sample', + type: 'table', + sql: 'CREATE TABLE sample(control TEXT, emoji TEXT, payload BLOB)', + }] as unknown as T[]); + } + if (sql.indexOf('FROM pragma_table_info') !== -1) { + return Promise.resolve([ + { cid: 0, name: LONG_COLUMN_NAME, type: 'TEXT', notnull: 0, dflt_value: null, pk: 0 }, + { cid: 1, name: 'emoji', type: 'TEXT', notnull: 0, dflt_value: null, pk: 0 }, + { cid: 2, name: 'payload', type: 'BLOB', notnull: 0, dflt_value: null, pk: 0 }, + ] as unknown as T[]); + } + if (sql.indexOf('SELECT COUNT(*) AS count') === 0) { + return Promise.resolve([{ count: 101 }] as unknown as T[]); + } + if (sql.indexOf('SELECT CASE WHEN typeof') === 0) { + return Promise.resolve(Array.from({ length: 101 }, () => ({ ...debugRow })) as unknown as T[]); + } + throw new Error(`Unexpected debug SQL: ${sql}`); + }, + }; +} + +describe('ClientSQLDebug', () => { + let capturedProvider: DebuggerProviderContract | undefined; + let dispose: jasmine.Spy<() => void>; + let notifyChange: jasmine.Spy<() => void>; + let ownerDispose: jasmine.Spy<() => void>; + let ownerModule: DebuggerProviderModule | undefined; + let ownerKey: string | undefined; + + beforeEach(() => { + capturedProvider = undefined; + ownerModule = undefined; + ownerKey = undefined; + dispose = jasmine.createSpy('dispose'); + notifyChange = jasmine.createSpy('notifyChange'); + let registrationDisposed = false; + const registration: DebuggerProviderRegistration = { + dispose(): void { + if (registrationDisposed) return; + registrationDisposed = true; + dispose(); + }, + notifyChange, + }; + ownerDispose = jasmine.createSpy('ownerDispose').and.callFake(() => registration.dispose()); + spyOn(DebuggerProvider, 'createDebuggerProviderOwner').and.callFake( + (creatingModule: DebuggerProviderModule, stableOwnerKey: string): DebuggerProviderOwner => { + ownerModule = creatingModule; + ownerKey = stableOwnerKey; + return { + dispose: ownerDispose, + register(provider: DebuggerProviderContract): DebuggerProviderRegistration { + capturedProvider = provider; + registrationDisposed = false; + return registration; + }, + }; + }, + ); + }); + + it('binds the module owner, bounds serialized actions, and disposes final registrations', async () => { + const database = makeDebugDatabase('clientsql_test_debug.sqlite'); + registerClientSQLDebugDatabase(database); + + expect(DebuggerProvider.createDebuggerProviderOwner).toHaveBeenCalledTimes(1); + expect(ownerModule).toBeDefined(); + expect(ownerKey).toBe('client_sql/src/ClientSQLDebug'); + const provider = capturedProvider as DebuggerProviderContract; + expect(provider.id).toBe('client-sql'); + expect(provider.kind).toBe(DebuggerProvider.DebuggerProviderKind.Sql); + expect(provider.availability?.()).toEqual(jasmine.objectContaining({ available: true })); + + const listDocument = (await provider.handleRequest({ action: 'list' })).json; + const listResponse = JSON.parse(listDocument) as Record; + expect(Array.isArray(listResponse.databases)).toBe(true); + expect(utf8ByteLength(listDocument)).toBeLessThanOrEqual(40 * 1024); + const listTruncation = listResponse.truncation as Record; + expect(listTruncation.payloadBytes) + .toBe(utf8ByteLength(listDocument)); + expect(listTruncation.omittedValues).toBe(1); + expect(listTruncation.reasons).toContain('collectionValues'); + expect((listTruncation.limits as Record).collectionItems).toBe(100); + + const listDatabases = listResponse.databases as Array>; + const databaseDebugInfo = listDatabases[0].debugInfo as Record; + const pendingChangedTables = databaseDebugInfo.pendingChangedTables as string[]; + expect(pendingChangedTables).toEqual(PENDING_CHANGED_TABLES.slice(0, 100)); + const databaseTruncation = listDatabases[0].truncation as Record; + expect(databaseTruncation.omittedValues).toBe(1); + expect(databaseTruncation.reasons).toContain('collectionValues'); + + const blobDocument = (await provider.handleRequest({ + action: 'table', + databaseId: `target:${database.id}`, + table: 'sample', + limit: 1, + offset: 0, + })).json; + expect(blobDocument).toContain('"encoding":"hex"'); + const blobResponse = JSON.parse(blobDocument) as Record; + const blobColumns = blobResponse.columns as Array>; + const blobRows = blobResponse.rows as unknown[][]; + expect((blobColumns[0].name as string).length).toBeGreaterThan(1024); + expect(Array.isArray(blobRows[0])).toBe(true); + expect(blobDocument).not.toContain(`"${LONG_COLUMN_NAME}":`); + + const tableDocument = (await provider.handleRequest({ + action: 'table', + databaseId: `target:${database.id}`, + table: 'sample', + limit: 100, + offset: 0, + })).json; + const tableResponse = JSON.parse(tableDocument) as Record; + expect(utf8ByteLength(tableDocument)).toBeLessThanOrEqual(40 * 1024); + expect((tableResponse.truncation as Record).payloadBytes) + .toBe(utf8ByteLength(tableDocument)); + expect(tableDocument).not.toContain('[object ArrayBuffer]'); + await expectAsync(provider.handleRequest({ action: 'write' }) as Promise) + .toBeRejectedWithError(/Unsupported ClientSQL debugger action/); + + notifyClientSQLDebugChanged(database.id); + expect(notifyChange).toHaveBeenCalled(); + unregisterClientSQLDebugDatabase(database); + expect(dispose).toHaveBeenCalledTimes(1); + expect(provider.availability?.()).toEqual(jasmine.objectContaining({ available: false })); + + const first = makeDebugDatabase('clientsql_first.sqlite'); + const second = makeDebugDatabase('clientsql_second.sqlite'); + registerClientSQLDebugDatabase(first); + registerClientSQLDebugDatabase(second); + + expect(DebuggerProvider.createDebuggerProviderOwner).toHaveBeenCalledTimes(1); + unregisterClientSQLDebugDatabase(first); + expect(dispose).toHaveBeenCalledTimes(1); + unregisterClientSQLDebugDatabase(second); + expect(dispose).toHaveBeenCalledTimes(2); + expect(ownerDispose).not.toHaveBeenCalled(); + }); +}); diff --git a/src/valdi_modules/src/valdi/client_sql/tsconfig.json b/src/valdi_modules/src/valdi/client_sql/tsconfig.json new file mode 100644 index 000000000..eeeaeca20 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../_configs/base.tsconfig.json", + "compilerOptions": { + "noImplicitReturns": true, + "noImplicitAny": true, + "strict": true + } +} diff --git a/src/valdi_modules/src/valdi/client_sql/web/ClientSQLNative.ts b/src/valdi_modules/src/valdi/client_sql/web/ClientSQLNative.ts new file mode 100644 index 000000000..2215ead97 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/web/ClientSQLNative.ts @@ -0,0 +1,77 @@ +import { + ClientSQLMigration, + ClientSQLNativeCallback, + ClientSQLNativeConnection, + ClientSQLNativeModule, + ClientSQLNativeTransactionBody, + ClientSQLValue, +} from '../src/ClientSQLNative'; + +class UnsupportedClientSQLConnection implements ClientSQLNativeConnection { + constructor( + private readonly name: string, + private readonly schemaVersion: number, + private readonly createStatements: string[], + private readonly migrations: ClientSQLMigration[], + ) {} + + execute( + _sql: string, + _parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + callback(undefined, unsupported(this.name, this.schemaVersion, this.createStatements, this.migrations).message); + } + + query( + _sql: string, + _parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + callback(undefined, unsupported(this.name, this.schemaVersion, this.createStatements, this.migrations).message); + } + + transaction( + _body: ClientSQLNativeTransactionBody, + callback: ClientSQLNativeCallback, + ): void { + callback(undefined, unsupported(this.name, this.schemaVersion, this.createStatements, this.migrations).message); + } + + close(callback: ClientSQLNativeCallback): void { + callback(undefined, undefined); + } +} + +function unsupported( + name: string, + schemaVersion: number, + createStatements: string[], + migrations: ClientSQLMigration[], +): Error { + return new Error( + `ClientSQLNative is not implemented for web. ` + + `Database '${name}' requested schema version ${schemaVersion} ` + + `with ${createStatements.length} create statements and ${migrations.length} migrations.`, + ); +} + +export const clientSQLNative: ClientSQLNativeModule = { + openDatabase( + name: string, + schemaVersion: number, + createStatements: string[], + migrations: ClientSQLMigration[], + ): ClientSQLNativeConnection { + return new UnsupportedClientSQLConnection(name, schemaVersion, createStatements, migrations); + }, +}; + +export function openDatabase( + name: string, + schemaVersion: number, + createStatements: string[], + migrations: ClientSQLMigration[], +): ClientSQLNativeConnection { + return clientSQLNative.openDatabase(name, schemaVersion, createStatements, migrations); +} diff --git a/src/valdi_modules/src/valdi/client_sql/web/tsconfig.json b/src/valdi_modules/src/valdi/client_sql/web/tsconfig.json new file mode 100644 index 000000000..5a14d6443 --- /dev/null +++ b/src/valdi_modules/src/valdi/client_sql/web/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2016", + "module": "commonjs", + "strict": true, + "composite": true, + "declaration": true, + "allowJs": true, + "lib": ["dom", "ES2019"], + "baseUrl": "..", + "paths": { + "valdi_core/*": ["../valdi_core/*"] + } + }, + "files": ["ClientSQLNative.ts", "../src/ClientSQLNative.d.ts"] +} diff --git a/third-party/sqlite/BUILD.bazel b/third-party/sqlite/BUILD.bazel new file mode 100644 index 000000000..89eac7888 --- /dev/null +++ b/third-party/sqlite/BUILD.bazel @@ -0,0 +1,11 @@ +# Package marker for the external SQLite repository build definition. + +filegroup( + name = "clientsql_generator_test_data", + srcs = [ + "sqlite.BUILD", + "sqlite_316.BUILD", + ], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/third-party/sqlite/LICENSE.md b/third-party/sqlite/LICENSE.md new file mode 100644 index 000000000..040435241 --- /dev/null +++ b/third-party/sqlite/LICENSE.md @@ -0,0 +1,85 @@ +License Information +=================== + +SQLite Is Public Domain +----------------------- + +The SQLite source code, including all of the files in the directories +listed in the bullets below are +[Public Domain](https://sqlite.org/copyright.html). +The authors have submitted written affidavits releasing their work to +the public for any use. Every byte of the public-domain code can be +traced back to the original authors. The files of this repository +that are public domain include the following: + + * All of the primary SQLite source code files found in the + [src/ directory](https://sqlite.org/src/tree/src?type=tree&expand) + * All of the test cases and testing code in the + [test/ directory](https://sqlite.org/src/tree/test?type=tree&expand) + * All of the SQLite extension source code and test cases in the + [ext/ directory](https://sqlite.org/src/tree/ext?type=tree&expand) + * All code that ends up in the "sqlite3.c" and "sqlite3.h" build products + that actually implement the SQLite RDBMS. + * All of the code used to compile the + [command-line interface](https://sqlite.org/cli.html) + * All of the code used to build various utility programs such as + "sqldiff", "sqlite3_rsync", and "sqlite3_analyzer". + + +The public domain source files usually contain a header comment +similar to the following to make it clear that the software is +public domain. + +> The author disclaims copyright to this source code. In place of +> a legal notice, here is a blessing: +> +> * May you do good and not evil. +> * May you find forgiveness for yourself and forgive others. +> * May you share freely, never taking more than you give. + +Almost every file you find in this source repository will be +public domain. But there are a small number of exceptions: + +Non-Public-Domain Code Included With This Source Repository AS A Convenience +---------------------------------------------------------------------------- + +This repository contains a (relatively) small amount of non-public-domain +code used to help implement the configuration and build logic. In other +words, there are some non-public-domain files used to implement: + +> ./configure && make + +In all cases, the non-public-domain files included with this +repository have generous BSD-style licenses. So anyone is free to +use any of the code in this source repository for any purpose, though +attribution may be required to reuse or republish the configure and +build scripts. None of the non-public-domain code ever actually reaches +the build products, such as "sqlite3.c", however, so no attribution is +required to use SQLite itself. The non-public-domain code consists of +scripts used to help compile SQLite. The non-public-domain code is +technically not part of SQLite. The non-public-domain code is +included in this repository as a convenience to developers, so that those +who want to build SQLite do not need to go download a bunch of +third-party build scripts in order to compile SQLite. + +Non-public-domain code included in this respository includes: + + * The ["autosetup"](http://msteveb.github.io/autosetup/) configuration + system that is contained (mostly) in the autosetup/ directory, but also + includes the "./configure" script at the top-level of this archive. + Autosetup has a separate BSD-style license. See the + [autosetup/LICENSE](http://msteveb.github.io/autosetup/license/) + for details. + + * There are BSD-style licenses on some of the configuration + software found in the legacy autoconf/ directory and its + subdirectories. + +The following unix shell command can be run from the top-level +of this source repository in order to remove all non-public-domain +code: + +> rm -rf configure autosetup autoconf + +If you unpack this source repository and then run the command above, what +is left will be 100% public domain. diff --git a/third-party/sqlite/README.md b/third-party/sqlite/README.md new file mode 100644 index 000000000..ad5d6b566 --- /dev/null +++ b/third-party/sqlite/README.md @@ -0,0 +1,44 @@ +# SQLite Dependency + +Non-Apple ClientSQL builds use SQLite 3.53.4 from Bazel's external `sqlite` +repository. The repository is declared in `bzl/dependencies.bzl` for WORKSPACE +consumers and `MODULE.bazel` for Bzlmod consumers. Apple builds use the system +SQLite framework instead. + +Source: + +- Official archive: `https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz` +- SHA-256: `0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c` + +`sqlite.BUILD` defines the portable C library over the downloaded amalgamation. +Android and default/Linux configurations compile and link that target so the +header and library come from the same hermetic dependency. SQLite remains +module-scoped and is linked only when a target depends on the `client_sql` Valdi +module. + +ClientSQL supports SQLite 3.16.0 and newer. That floor covers the table-valued +PRAGMA support used by the generic debugger inspector; WAL itself predates the +floor. Every native connection checks `sqlite3_libversion_number()` before use, +and writer opens execute `PRAGMA journal_mode = WAL` and verify that SQLite +actually returned `wal` rather than assuming the requested mode was accepted. + +## Generator validation floor + +SQL source compatibility is checked by the separately named `sqlite_316` +repository and `//compiler/clientsql:sqlite_316_validator`. It is not linked +into the ClientSQL runtime. The validator is built against this primary SQLite +artifact: + +- Official archive: `https://www.sqlite.org/2017/sqlite-amalgamation-3160000.zip` +- Official release identity: `https://sqlite.org/releaselog/3_16_0.html` +- SHA-256: `3b5dfb65807e2b17e6463357df848e322badba01dc9a4a1de8fdbb72d448e3b0` +- SQLite version: `3.16.0` +- `SQLITE_SOURCE_ID`: `2017-01-02 11:57:58 04ac0b75b1716541b2b97704f4809cb7ef19cccf` +- SHA-1 of `sqlite3.c`: `e2920fb885569d14197c9b7958e6f1db573ee669` +- License: SQLite public-domain dedication (`LICENSE.md` in this directory) + +The version, source ID, and `sqlite3.c` SHA-1 match SQLite's official 3.16.0 +release log. Both Bzlmod and WORKSPACE declarations pin the downloaded archive +by SHA-256. The validator checks its linked SQLite identity on every invocation; +the generator then checks the validator's declared identity and incorporates +the executable's SHA-256 into `clientsql -version` and compiler cache metadata. diff --git a/third-party/sqlite/sqlite.BUILD b/third-party/sqlite/sqlite.BUILD new file mode 100644 index 000000000..1b1bd1fe6 --- /dev/null +++ b/third-party/sqlite/sqlite.BUILD @@ -0,0 +1,24 @@ +cc_library( + name = "sqlite", + srcs = ["sqlite3.c"], + hdrs = [ + "sqlite3.h", + "sqlite3ext.h", + ], + copts = [ + "-DSQLITE_DEFAULT_MEMSTATUS=0", + "-DSQLITE_OMIT_LOAD_EXTENSION", + "-DSQLITE_THREADSAFE=1", + "-Wno-cast-qual", + "-Wno-implicit-fallthrough", + "-Wno-pedantic", + "-Wno-shorten-64-to-32", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Wno-unknown-warning-option", + ], + strip_include_prefix = ".", + visibility = ["//visibility:public"], +) diff --git a/third-party/sqlite/sqlite_316.BUILD b/third-party/sqlite/sqlite_316.BUILD new file mode 100644 index 000000000..e7920f5f1 --- /dev/null +++ b/third-party/sqlite/sqlite_316.BUILD @@ -0,0 +1,27 @@ +cc_library( + name = "sqlite", + srcs = ["sqlite3.c"], + hdrs = [ + "sqlite3.h", + "sqlite3ext.h", + ], + copts = [ + "-DSQLITE_DEFAULT_MEMSTATUS=0", + "-DSQLITE_OMIT_LOAD_EXTENSION", + "-DSQLITE_THREADSAFE=0", + "-Wno-cast-qual", + "-Wno-implicit-fallthrough", + "-Wno-implicit-const-int-float-conversion", + "-Wno-implicit-function-declaration", + "-Wno-pedantic", + "-Wno-shorten-64-to-32", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Wno-unknown-warning-option", + "-Wno-deprecated-declarations", + ], + strip_include_prefix = ".", + visibility = ["//visibility:public"], +) diff --git a/valdi/BUILD.bazel b/valdi/BUILD.bazel index 2e89e1b01..a3ee369e7 100644 --- a/valdi/BUILD.bazel +++ b/valdi/BUILD.bazel @@ -903,9 +903,10 @@ valdi_static_resource( valdi_test( name = "test_integration", - srcs = glob([ - "test/integration/**/*.cpp", - ]) + [":remote_assets_static_res"], + srcs = glob( + ["test/integration/**/*.cpp"], + exclude = ["test/integration/ClientSQLRuntime_tests.cpp"], + ) + [":remote_assets_static_res"], hdrs = glob(["test/integration/**/*.hpp"]), deps = [ ":test_utils", @@ -925,6 +926,30 @@ valdi_test( ], ) +# Runs the generated TestDb TypeScript through a real Valdi JS runtime while +# linking ClientSQL's self-registering native module factory. This intentionally +# remains a distinct target so the optional SQLite dependency does not expand +# every Valdi integration test binary. +valdi_test( + name = "test_client_sql_runtime_integration", + srcs = [ + "test/integration/ClientSQLRuntime_tests.cpp", + "test/integration/JSBridgeTestFixture.cpp", + "test/integration/RuntimeTestsUtils.cpp", + ], + hdrs = [ + "test/integration/JSBridgeTestFixture.hpp", + "test/integration/RuntimeTestsUtils.hpp", + ], + deps = [ + ":test_utils", + ":valdi_runtime_with_vm", + "//src/valdi_modules/src/valdi/valdi_core:valdi_core_native", + "//tsn", + "//valdi/testdata/resources/modules/client_sql_smoke:client_sql_smoke_native_desktop", + ], +) + valdi_test( name = "test_layout", size = "large", @@ -1270,3 +1295,13 @@ filegroup( ]), visibility = ["//visibility:public"], ) + +filegroup( + name = "clientsql_generator_test_data", + srcs = [ + "BUILD.bazel", + "test/integration/ClientSQLRuntime_tests.cpp", + ], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/valdi/test/integration/ClientSQLRuntime_tests.cpp b/valdi/test/integration/ClientSQLRuntime_tests.cpp new file mode 100644 index 000000000..c7005697e --- /dev/null +++ b/valdi/test/integration/ClientSQLRuntime_tests.cpp @@ -0,0 +1,126 @@ +#include "JSBridgeTestFixture.hpp" +#include "RuntimeTestsUtils.hpp" +#include "gtest/gtest.h" +#include "valdi/runtime/Resources/DiskCacheImpl.hpp" +#include "valdi/runtime/Resources/ResourceManager.hpp" + +#include +#include +#include +#include +#include + +using namespace Valdi; + +namespace ValdiTest { + +class ClientSQLRuntimeTemporaryDirectory { +public: + ClientSQLRuntimeTemporaryDirectory() { + char path[] = "/tmp/valdi-clientsql-runtime-XXXXXX"; + const auto directory = mkdtemp(path); + if (directory != nullptr) { + _path = directory; + } + } + + ~ClientSQLRuntimeTemporaryDirectory() { + if (!_path.empty()) { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + } + + const std::string& path() const { + return _path; + } + +private: + std::string _path; +}; + +class ClientSQLRuntimeFixture : public JSBridgeTestFixture { +protected: + void SetUp() override { + JSBridgeTestFixture::SetUp(); + if (IsSkipped()) { + return; + } + ASSERT_FALSE(directory.path().empty()); + auto diskCache = makeShared(StringCache::getGlobal().makeString(directory.path())); + wrapper = RuntimeWrapper( + getJsBridge(), + isWithTSN() ? TSNMode::Enabled : TSNMode::Disabled, + diskCache); + } + + void TearDown() override { + wrapper.teardown(); + } + + ClientSQLRuntimeTemporaryDirectory directory; + RuntimeWrapper wrapper; +}; + +TEST_P(ClientSQLRuntimeFixture, resolvesGeneratedDatabaseThroughRealNativeModule) { + auto loadResult = wrapper.loadModule( + STRING_LITERAL("client_sql_smoke"), + ResourceManagerLoadModuleType::Sources); + ASSERT_TRUE(loadResult) << loadResult.description(); + + const std::string startScript = R"JS( + global.__clientSQLRuntimeDone = false; + global.__clientSQLRuntimeResult = undefined; + global.__clientSQLRuntimeError = undefined; + global.require('client_sql_smoke/src/ClientSQLSmoke').runClientSQLNativeIntegration( + function(result, error) { + global.__clientSQLRuntimeResult = result; + global.__clientSQLRuntimeError = error; + global.__clientSQLRuntimeDone = true; + } + ); + )JS"; + auto startResult = wrapper.runtime->getJavaScriptRuntime()->evaluateScript( + makeShared(startScript)->toBytesView(), + STRING_LITERAL("clientsql_runtime_integration_start.js")); + ASSERT_TRUE(startResult) << startResult.description(); + + bool completed = false; + for (int attempt = 0; attempt < 500; ++attempt) { + wrapper.flushQueues(); + const std::string doneScript = "return global.__clientSQLRuntimeDone === true;"; + auto doneResult = wrapper.runtime->getJavaScriptRuntime()->evaluateScript( + makeShared(doneScript)->toBytesView(), + STRING_LITERAL("clientsql_runtime_integration_poll.js")); + ASSERT_TRUE(doneResult) << doneResult.description(); + if (doneResult.value().toBool()) { + completed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + ASSERT_TRUE(completed) << "Timed out waiting for generated ClientSQL/native integration"; + + const std::string errorScript = "return global.__clientSQLRuntimeError || '';"; + auto errorResult = wrapper.runtime->getJavaScriptRuntime()->evaluateScript( + makeShared(errorScript)->toBytesView(), + STRING_LITERAL("clientsql_runtime_integration_error.js")); + ASSERT_TRUE(errorResult) << errorResult.description(); + EXPECT_EQ(STRING_LITERAL(""), errorResult.value().toStringBox()); + + const std::string resultScript = "return global.__clientSQLRuntimeResult || '';"; + auto integrationResult = wrapper.runtime->getJavaScriptRuntime()->evaluateScript( + makeShared(resultScript)->toBytesView(), + STRING_LITERAL("clientsql_runtime_integration_result.js")); + ASSERT_TRUE(integrationResult) << integrationResult.description(); + EXPECT_EQ(STRING_LITERAL("ok"), integrationResult.value().toStringBox()); +} + +INSTANTIATE_TEST_SUITE_P(ClientSQLRuntimeTests, + ClientSQLRuntimeFixture, + ::testing::Values(JavaScriptEngineTestCase::Hermes, + JavaScriptEngineTestCase::QuickJS, + JavaScriptEngineTestCase::JSCore), + PrintJavaScriptEngineType()); + +} // namespace ValdiTest diff --git a/valdi/test/integration/RuntimeTestsUtils.cpp b/valdi/test/integration/RuntimeTestsUtils.cpp index cd86df703..5482f3aa7 100644 --- a/valdi/test/integration/RuntimeTestsUtils.cpp +++ b/valdi/test/integration/RuntimeTestsUtils.cpp @@ -9,6 +9,11 @@ RuntimeWrapper::RuntimeWrapper() = default; RuntimeWrapper::RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, TSNMode tsnMode) : RuntimeWrapper(jsBridge, tsnMode, false) {} +RuntimeWrapper::RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, + TSNMode tsnMode, + const Ref& runtimeDiskCache) + : RuntimeWrapper(jsBridge, tsnMode, false, nullptr, runtimeDiskCache) {} + RuntimeWrapper::RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, TSNMode tsnMode, bool enableViewPreloader) : RuntimeWrapper(jsBridge, tsnMode, enableViewPreloader, nullptr) {} @@ -16,6 +21,13 @@ RuntimeWrapper::RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, TSNMode tsnMode, bool enableViewPreloader, const Shared& tweakValueProvider) + : RuntimeWrapper(jsBridge, tsnMode, enableViewPreloader, tweakValueProvider, nullptr) {} + +RuntimeWrapper::RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, + TSNMode tsnMode, + bool enableViewPreloader, + const Shared& tweakValueProvider, + const Ref& runtimeDiskCache) : logger(&Valdi::ConsoleLogger::getLogger()), mainQueue(Valdi::makeShared()), runtimeListener(Valdi::makeShared()), @@ -30,7 +42,7 @@ RuntimeWrapper::RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, true, jsBridge, mainQueue, - diskCache, + runtimeDiskCache != nullptr ? runtimeDiskCache : diskCache, runtimeListener, resourceLoader, tweakValueProvider); @@ -226,4 +238,4 @@ Valdi::Result RuntimeWrapper::loadModule(const Valdi::StringBox& bu return resultHolder->waitForResult(); } -} // namespace ValdiTest \ No newline at end of file +} // namespace ValdiTest diff --git a/valdi/test/integration/RuntimeTestsUtils.hpp b/valdi/test/integration/RuntimeTestsUtils.hpp index 19624523a..fd229a1d5 100644 --- a/valdi/test/integration/RuntimeTestsUtils.hpp +++ b/valdi/test/integration/RuntimeTestsUtils.hpp @@ -27,6 +27,10 @@ struct RuntimeWrapper { RuntimeWrapper(); RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, TSNMode tsnMode); + RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, + TSNMode tsnMode, + const Valdi::Ref& runtimeDiskCache); + RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, TSNMode tsnMode, bool enableViewPreloader); RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, @@ -34,6 +38,12 @@ struct RuntimeWrapper { bool enableViewPreloader, const Valdi::Shared& tweakValueProvider); + RuntimeWrapper(Valdi::IJavaScriptBridge* jsBridge, + TSNMode tsnMode, + bool enableViewPreloader, + const Valdi::Shared& tweakValueProvider, + const Valdi::Ref& runtimeDiskCache); + ~RuntimeWrapper(); void teardown(); @@ -74,4 +84,4 @@ struct RuntimeWrapper { Valdi::ResourceManagerLoadModuleType loadType); }; -} // namespace ValdiTest \ No newline at end of file +} // namespace ValdiTest diff --git a/valdi/testdata/resources/modules/client_sql_smoke/BUILD.bazel b/valdi/testdata/resources/modules/client_sql_smoke/BUILD.bazel new file mode 100644 index 000000000..5740ec35b --- /dev/null +++ b/valdi/testdata/resources/modules/client_sql_smoke/BUILD.bazel @@ -0,0 +1,33 @@ +load("@valdi//bzl/valdi:valdi_module.bzl", "valdi_module") + +valdi_module( + name = "client_sql_smoke", + srcs = glob([ + "src/**/*.ts", + "src/**/*.tsx", + ]) + [ + "tsconfig.json", + ], + android_output_target = "release", + ios_module_name = "SCCClientSQLSmoke", + ios_output_target = "release", + sql_db_names = ["TestDb"], + sql_srcs = glob([ + "sql/**/*.sq", + "sql/**/*.sqm", + "sql/sql_types.yaml", + "sql/sql_manifest.yaml", + ]), + visibility = ["//visibility:public"], + deps = [ + "//src/valdi_modules/src/valdi/client_sql", + "//src/valdi_modules/src/valdi/valdi_core", + ], +) + +filegroup( + name = "clientsql_generator_test_data", + srcs = ["src/ClientSQLSmoke.ts"], + testonly = True, + visibility = ["//compiler/clientsql:__pkg__"], +) diff --git a/valdi/testdata/resources/modules/client_sql_smoke/module.yaml b/valdi/testdata/resources/modules/client_sql_smoke/module.yaml new file mode 100644 index 000000000..8a2b84aff --- /dev/null +++ b/valdi/testdata/resources/modules/client_sql_smoke/module.yaml @@ -0,0 +1,9 @@ +name: client_sql_smoke +ios: + module_name: SCCClientSQLSmoke + output: release +android: + output: release +dependencies: + - client_sql + - valdi_core diff --git a/valdi/testdata/resources/modules/client_sql_smoke/sql/TestDb/User.sq b/valdi/testdata/resources/modules/client_sql_smoke/sql/TestDb/User.sq new file mode 100644 index 000000000..dc0c2dad0 --- /dev/null +++ b/valdi/testdata/resources/modules/client_sql_smoke/sql/TestDb/User.sq @@ -0,0 +1,29 @@ +CREATE TABLE user ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + age INTEGER, + nickname TEXT +); + +CREATE TABLE runtime_value ( + id INTEGER NOT NULL PRIMARY KEY, + enabled BOOLEAN NOT NULL, + optional BOOLEAN, + payload BLOB NOT NULL +); + +selectAll: +SELECT * FROM user; + +selectById: +SELECT id, name, age FROM user WHERE id = :id; + +insertUser: +INSERT INTO user(id, name, age) VALUES (:id, :name, :age); + +insertRuntimeValue: +INSERT INTO runtime_value(id, enabled, optional, payload) +VALUES (:id, :enabled, :optional, :payload); + +selectRuntimeValues: +SELECT id, enabled, optional, payload FROM runtime_value ORDER BY id; diff --git a/valdi/testdata/resources/modules/client_sql_smoke/sql/migration/2.sqm b/valdi/testdata/resources/modules/client_sql_smoke/sql/migration/2.sqm new file mode 100644 index 000000000..f745423c5 --- /dev/null +++ b/valdi/testdata/resources/modules/client_sql_smoke/sql/migration/2.sqm @@ -0,0 +1 @@ +ALTER TABLE user ADD COLUMN nickname TEXT; diff --git a/valdi/testdata/resources/modules/client_sql_smoke/src/ClientSQLSmoke.ts b/valdi/testdata/resources/modules/client_sql_smoke/src/ClientSQLSmoke.ts new file mode 100644 index 000000000..649f1185d --- /dev/null +++ b/valdi/testdata/resources/modules/client_sql_smoke/src/ClientSQLSmoke.ts @@ -0,0 +1,116 @@ +import { + ClientSQLMigration, + ClientSQLNativeCallback, + ClientSQLNativeConnection, + ClientSQLNativeModule, + ClientSQLNativeTransaction, + ClientSQLNativeTransactionBody, + ClientSQLValue, +} from 'client_sql/src/ClientSQLNative'; +import { TestDb, setClientSQLNativeForTests } from './sqlgen/TestDb'; + +/** + * @Version(__PLACEHOLDER__) + */ +class FakeConnection implements ClientSQLNativeConnection { + readonly executed: string[] = []; + + execute( + sql: string, + _parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + this.executed.push(sql); + callback(undefined, undefined); + } + + query( + _sql: string, + _parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + callback([], undefined); + } + + transaction(body: ClientSQLNativeTransactionBody, callback: ClientSQLNativeCallback): void { + const transaction: ClientSQLNativeTransaction = { + execute: (sql, parameters, transactionCallback): void => { + this.execute(sql, parameters, transactionCallback); + }, + query: ( + sql: string, + parameters: ClientSQLValue[] | undefined, + transactionCallback: ClientSQLNativeCallback, + ): void => { + this.query(sql, parameters, transactionCallback); + }, + }; + body(transaction, callback); + } + + close(callback: ClientSQLNativeCallback): void { + callback(undefined, undefined); + } +} + +export function createSmokeDatabase(): TestDb { + const native: ClientSQLNativeModule = { + /** + * @Version(__PLACEHOLDER__) + */ + openDatabase( + _name: string, + _schemaVersion: number, + _createStatements: string[], + _migrations: ClientSQLMigration[], + ): ClientSQLNativeConnection { + return new FakeConnection(); + }, + }; + + setClientSQLNativeForTests(native); + return TestDb.open(undefined); +} + +export async function runClientSQLSmoke(): Promise { + const db = createSmokeDatabase(); + await db.userQueries.insertUser(1, 'Ada', null); + await db.userQueries.selectById(1); + await db.close(); +} + +export type ClientSQLRuntimeIntegrationCallback = ( + result: string | undefined, + error: string | undefined, +) => void; + +export function runClientSQLNativeIntegration(callback: ClientSQLRuntimeIntegrationCallback): void { + void runClientSQLNativeIntegrationAsync().then( + result => callback(result, undefined), + error => callback(undefined, error instanceof Error ? error.message : String(error)), + ); +} + +async function runClientSQLNativeIntegrationAsync(): Promise { + const db = TestDb.open(`runtime-integration-${Date.now()}`); + try { + await db.userQueries.insertRuntimeValue(1, true, null, new ArrayBuffer(0)); + await db.userQueries.insertRuntimeValue(2, false, true, new ArrayBuffer(0)); + const rows = await db.userQueries.selectRuntimeValues(); + if (rows.length !== 2) { + throw new Error(`Expected two runtime rows, received ${rows.length}`); + } + if (rows[0].enabled !== true || rows[0].optional !== null) { + throw new Error('Native ClientSQL did not preserve true/null boolean values.'); + } + if (rows[1].enabled !== false || rows[1].optional !== true) { + throw new Error('Native ClientSQL did not preserve false/true boolean values.'); + } + if (rows[0].payload.byteLength !== 0 || rows[1].payload.byteLength !== 0) { + throw new Error('Native ClientSQL did not preserve zero-length BLOB values.'); + } + return 'ok'; + } finally { + await db.close(); + } +} diff --git a/valdi/testdata/resources/modules/client_sql_smoke/tsconfig.json b/valdi/testdata/resources/modules/client_sql_smoke/tsconfig.json new file mode 100644 index 000000000..081f938b1 --- /dev/null +++ b/valdi/testdata/resources/modules/client_sql_smoke/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../../../../src/valdi_modules/src/valdi/_configs/base.tsconfig.json", + "compilerOptions": { + "noImplicitReturns": true, + "noImplicitAny": true, + "strict": true + } +}