diff --git a/compiler/clientsql/BUILD.bazel b/compiler/clientsql/BUILD.bazel new file mode 100644 index 00000000..3d694ea1 --- /dev/null +++ b/compiler/clientsql/BUILD.bazel @@ -0,0 +1,50 @@ +load("@aspect_rules_js//js:defs.bzl", "js_binary") +load("@rules_python//python:defs.bzl", "py_test") + +py_library( + name = "clientsql_lib", + srcs = glob(["src/clientsql/*.py"]), + imports = ["src"], + visibility = ["//visibility:private"], +) + +py_binary( + name = "clientsql", + srcs = ["src/clientsql_main.py"], + main = "src/clientsql_main.py", + visibility = ["//visibility:public"], + deps = [":clientsql_lib"], +) + +py_binary( + name = "package_clientsql", + srcs = ["package_clientsql.py"], + data = glob(["src/clientsql/*.py"]), + main = "package_clientsql.py", + visibility = ["//visibility:private"], +) + +js_binary( + name = "clientsql_test_javascript_runner", + entry_point = "run_generated_javascript.js", + testonly = True, +) + +py_test( + name = "test_clientsql", + srcs = ["test_clientsql.py"], + data = [ + ":clientsql", + ":clientsql_test_javascript_runner", + ":package_clientsql", + "@npm_typescript//:tsc", + ], + env = { + "BAZEL_BINDIR": ".", + "CLIENTSQL_TEST_GENERATOR": "$(rootpath :clientsql)", + "CLIENTSQL_TEST_JAVASCRIPT_RUNNER": "$(rootpath :clientsql_test_javascript_runner)", + "CLIENTSQL_TEST_TYPESCRIPT_COMPILER": "$(rootpath @npm_typescript//:tsc)", + }, + size = "medium", + deps = [":clientsql_lib"], +) diff --git a/compiler/clientsql/README.md b/compiler/clientsql/README.md new file mode 100644 index 00000000..65c28758 --- /dev/null +++ b/compiler/clientsql/README.md @@ -0,0 +1,27 @@ +# ClientSQL generator + +This directory contains the canonical, reviewable Python source for Valdi's +SQLDelight-style ClientSQL generator. + +The source is divided by responsibility: + +- `cli.py` owns command-line parsing and generation orchestration. +- `model.py` defines the schema and query model. +- `sql.py` parses and validates SQL, migrations, parameters, and result shapes. +- `typescript.py` emits generated TypeScript bindings and database classes. + +The public Valdi toolchain continues to supply its ClientSQL executable through +the existing `sqldelight_compiler` target. This source package intentionally +does not replace that toolchain binary or check in a generated executable. Use +the Bazel `//compiler/clientsql:clientsql` target for source builds, or create a +deterministic standalone zipapp at an explicit local path: + +```bash +python3 compiler/clientsql/package_clientsql.py --output /tmp/clientsql +``` + +An explicitly supplied executable can be checked against the canonical source: + +```bash +python3 compiler/clientsql/package_clientsql.py --output /tmp/clientsql --check +``` diff --git a/compiler/clientsql/package_clientsql.py b/compiler/clientsql/package_clientsql.py new file mode 100644 index 00000000..9f6d93b9 --- /dev/null +++ b/compiler/clientsql/package_clientsql.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Sequence + + +SOURCE_ROOT = Path(__file__).resolve().parent / "src" +FIXED_ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +ENTRYPOINT = "from clientsql.cli import entrypoint\n\nentrypoint()\n" + + +def write_zip_entry(archive: zipfile.ZipFile, name: str, content: bytes) -> None: + info = zipfile.ZipInfo(name, date_time=FIXED_ZIP_TIMESTAMP) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o100644 << 16 + archive.writestr(info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + + +def package_clientsql(output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=output.parent, prefix=f".{output.name}.", delete=False) as temporary: + temporary_path = Path(temporary.name) + temporary.write(b"#!/usr/bin/env python3\n") + + try: + with zipfile.ZipFile(temporary_path, mode="a") as archive: + write_zip_entry(archive, "__main__.py", ENTRYPOINT.encode("utf-8")) + for source_path in sorted(SOURCE_ROOT.rglob("*.py")): + archive_path = source_path.relative_to(SOURCE_ROOT).as_posix() + write_zip_entry(archive, archive_path, source_path.read_bytes()) + os.chmod(temporary_path, 0o755) + temporary_path.replace(output) + finally: + temporary_path.unlink(missing_ok=True) + + +def main(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser(description="Package the modular ClientSQL generator as one executable zipapp") + parser.add_argument( + "--output", + type=Path, + required=True, + help="Path for the generated executable zipapp", + ) + parser.add_argument( + "--check", + action="store_true", + help="Verify that the supplied executable matches the canonical source", + ) + args = parser.parse_args(argv) + + if not args.check: + package_clientsql(args.output) + return 0 + + with tempfile.TemporaryDirectory(prefix="clientsql-package-check-") as temporary_directory: + candidate = Path(temporary_directory) / "clientsql" + package_clientsql(candidate) + if not args.output.is_file() or candidate.read_bytes() != args.output.read_bytes(): + print( + f"{args.output} is stale; rerun this command without --check", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/compiler/clientsql/run_generated_javascript.js b/compiler/clientsql/run_generated_javascript.js new file mode 100644 index 00000000..c2ee5f52 --- /dev/null +++ b/compiler/clientsql/run_generated_javascript.js @@ -0,0 +1,9 @@ +'use strict'; + +const path = require('node:path'); + +if (process.argv.length !== 3) { + throw new Error('Expected one generated JavaScript entrypoint'); +} + +require(path.resolve(process.argv[2])); diff --git a/compiler/clientsql/src/clientsql/__init__.py b/compiler/clientsql/src/clientsql/__init__.py new file mode 100644 index 00000000..5a25da4f --- /dev/null +++ b/compiler/clientsql/src/clientsql/__init__.py @@ -0,0 +1 @@ +"""Typed SQLite code generation for Valdi ClientSQL.""" diff --git a/compiler/clientsql/src/clientsql/__main__.py b/compiler/clientsql/src/clientsql/__main__.py new file mode 100644 index 00000000..e0da6ff6 --- /dev/null +++ b/compiler/clientsql/src/clientsql/__main__.py @@ -0,0 +1,4 @@ +from .cli import entrypoint + + +entrypoint() diff --git a/compiler/clientsql/src/clientsql/cli.py b/compiler/clientsql/src/clientsql/cli.py new file mode 100644 index 00000000..9857ed2d --- /dev/null +++ b/compiler/clientsql/src/clientsql/cli.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Optional, Sequence + +from .model import ClientSqlError +from .sql import ( + collect_create_statements, + collect_migrations, + collect_tables, + load_type_mapping, + parse_sql_file, + sanitize_type_name, + validate_schema_and_queries, +) +from .typescript import write_database_file, write_queries_file, write_types_file + + +VERSION = "valdi-clientsql 0.2.0" + + +def main(argv: Sequence[str]) -> int: + if "-version" in argv or "--version" in argv: + print(VERSION) + return 0 + + parser = argparse.ArgumentParser(prog="clientsql") + parser.add_argument("-s", "--source", required=True, help="SQL source directory") + parser.add_argument("-p", "--package", required=True, help="Database package/name") + parser.add_argument("-c", "--class", dest="class_name", required=True, help="Database class name") + parser.add_argument("-m", "--module", required=True, help="Module name") + parser.add_argument("-o", "--output", required=True, help="Output directory") + parser.add_argument("-l", "--language", required=True, choices=["typescript"], help="Output language") + parser.add_argument("-tm", "--type-mapping", dest="type_mapping", help="Optional sql_types.yaml") + args = parser.parse_args(argv) + + try: + generate( + sql_dir=Path(args.source), + package_name=args.package, + class_name=args.class_name, + output_dir=Path(args.output), + type_mapping=args.type_mapping, + ) + except ClientSqlError as exc: + print(f"ClientSQL error: {exc}", file=sys.stderr) + return 1 + + return 0 + + +def generate( + sql_dir: Path, + package_name: str, + class_name: str, + output_dir: Path, + type_mapping: Optional[str], +) -> None: + package_dir = sql_dir / package_name + if not package_dir.is_dir(): + raise ClientSqlError(f"SQL package directory does not exist: {package_dir}") + + output_dir.mkdir(parents=True, exist_ok=True) + custom_types = load_type_mapping(sql_dir, type_mapping) + sql_paths = sorted(package_dir.rglob("*.sq")) + if not sql_paths: + raise ClientSqlError(f"No .sq files found under {package_dir}") + + sql_text_by_path = {path: path.read_text(encoding="utf-8") for path in sql_paths} + tables = collect_tables(sql_text_by_path.values(), custom_types) + sql_files = [ + parse_sql_file(path, package_dir, tables) + for path in sql_paths + ] + + create_statements = collect_create_statements(sql_text_by_path.values()) + migrations = collect_migrations(sql_dir) + validate_schema_and_queries(create_statements, sql_files) + + for sql_file in sql_files: + write_types_file(output_dir, sql_file, tables) + write_queries_file(output_dir, sql_file) + + write_database_file( + output_dir=output_dir, + class_name=sanitize_type_name(class_name), + db_name=package_name, + sql_files=sql_files, + create_statements=create_statements, + migrations=migrations, + ) + + +def entrypoint() -> None: + sys.exit(main(sys.argv[1:])) diff --git a/compiler/clientsql/src/clientsql/model.py b/compiler/clientsql/src/clientsql/model.py new file mode 100644 index 00000000..c4ff54e0 --- /dev/null +++ b/compiler/clientsql/src/clientsql/model.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import List + + +@dataclass +class Column: + name: str + sql_type: str + ts_type: str + nullable: bool + + +@dataclass +class Table: + name: str + columns: List[Column] + + +@dataclass +class Parameter: + name: str + ts_type: str + + +@dataclass +class ParamOccurrence: + start: int + end: int + name: str + nullable: bool + + +@dataclass +class Query: + name: str + sql: str + runtime_sql: str + param_order: List[str] + params: List[Parameter] + result_type: str + result_fields: List[Column] + returns_rows: bool + read_tables: List[str] + changed_tables: List[str] + + +@dataclass +class SqlFile: + path: Path + rel_to_package: Path + stem_path: Path + queries: List[Query] + + +class ClientSqlError(Exception): + pass diff --git a/compiler/clientsql/src/clientsql/sql.py b/compiler/clientsql/src/clientsql/sql.py new file mode 100644 index 00000000..ab9f16ca --- /dev/null +++ b/compiler/clientsql/src/clientsql/sql.py @@ -0,0 +1,680 @@ +from __future__ import annotations + +import re +import sqlite3 +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + +from .model import Column, ParamOccurrence, Parameter, Query, SqlFile, Table, ClientSqlError + + +SQL_IDENTIFIER_PATTERN = r"[`\"\[]?[A-Za-z_][A-Za-z0-9_]*[`\"\]]?" + + +def load_type_mapping(sql_dir: Path, type_mapping: Optional[str]) -> Dict[str, str]: + if not type_mapping: + return {} + + mapping_path = Path(type_mapping) + if not mapping_path.is_absolute(): + mapping_path = sql_dir / mapping_path + + if not mapping_path.is_file(): + raise ClientSqlError(f"Type mapping file does not exist: {mapping_path}") + + result: Dict[str, str] = {} + current_key: Optional[str] = None + for raw_line in mapping_path.read_text(encoding="utf-8").splitlines(): + line = raw_line.rstrip() + if not line.strip() or line.lstrip().startswith("#"): + continue + top_level = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)\s*:\s*$", line) + if top_level: + current_key = top_level.group(1).upper() + continue + if current_key is None: + continue + type_match = re.search(r"\b(?:typescript|ts|type)\s*:\s*['\"]?([^'\"]+)['\"]?", line, re.IGNORECASE) + if type_match: + result[current_key] = type_match.group(1).strip() + return result + + +def collect_tables(sql_texts: Iterable[str], custom_types: Dict[str, str]) -> Dict[str, Table]: + tables: Dict[str, Table] = {} + for sql_text in sql_texts: + for match in re.finditer( + r"\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([`\"A-Za-z_][`\"A-Za-z0-9_]*)\s*\((.*?)\)\s*;", + strip_sql_comments(sql_text), + re.IGNORECASE | re.DOTALL, + ): + table_name = clean_identifier(match.group(1)) + body = match.group(2) + columns = parse_columns(body, custom_types) + if columns: + tables[table_name.lower()] = Table(name=table_name, columns=columns) + return tables + + +def collect_create_statements(sql_texts: Iterable[str]) -> List[str]: + table_statements: List[str] = [] + other_statements: List[str] = [] + for sql_text in sql_texts: + for statement in split_sql_statements(schema_prefix(sql_text)): + if re.match( + r"^\s*CREATE\s+(?:UNIQUE\s+)?(?:TABLE|INDEX|VIEW|VIRTUAL\s+TABLE)\b", + statement, + re.IGNORECASE, + ): + target = table_statements if re.match( + r"^\s*CREATE\s+(?:VIRTUAL\s+)?TABLE\b", statement, re.IGNORECASE + ) else other_statements + target.append(statement.strip()) + return table_statements + other_statements + + +def schema_prefix(sql_text: str) -> str: + lines: List[str] = [] + for raw_line in sql_text.splitlines(): + if re.match(r"^\s*[A-Za-z_][A-Za-z0-9_]*\s*:\s*", raw_line) and not raw_line.lstrip().startswith("--"): + break + lines.append(raw_line) + return "\n".join(lines) + + +def collect_migrations(sql_dir: Path) -> List[Tuple[int, List[str]]]: + migration_dir = sql_dir / "migration" + if not migration_dir.is_dir(): + return [] + + migrations: List[Tuple[int, List[str]]] = [] + seen_versions: set[int] = set() + for path in sorted(migration_dir.glob("*.sqm"), key=migration_sort_key): + version_match = re.match(r"(\d+)", path.stem) + if version_match is None: + raise ClientSqlError(f"Migration filename must start with a version number: {path}") + version = int(version_match.group(1)) + if version in seen_versions: + raise ClientSqlError(f"Duplicate migration version {version}: {path}") + seen_versions.add(version) + statements = [statement.strip() for statement in split_sql_statements(path.read_text(encoding="utf-8"))] + migrations.append((version, statements)) + + if migrations: + versions = [version for version, _ in migrations] + expected_versions = list(range(2, versions[-1] + 1)) + if versions != expected_versions: + raise ClientSqlError( + f"Migration versions must be contiguous starting at 2; found {versions}" + ) + return migrations + + +def validate_schema_and_queries(create_statements: Sequence[str], sql_files: Sequence[SqlFile]) -> None: + database = sqlite3.connect(":memory:") + try: + database.execute("PRAGMA foreign_keys = ON") + for statement in create_statements: + try: + database.execute(statement) + except sqlite3.Error as exc: + raise ClientSqlError(f"Invalid schema statement: {exc}\n{statement}") from exc + + for sql_file in sql_files: + for query in sql_file.queries: + try: + database.execute(f"EXPLAIN {query.runtime_sql}", [None] * len(query.param_order)) + except sqlite3.Error as exc: + raise ClientSqlError( + f"Invalid query {sql_file.rel_to_package}:{query.name}: {exc}" + ) from exc + finally: + database.close() + + +def migration_sort_key(path: Path) -> Tuple[int, str]: + match = re.match(r"(\d+)", path.stem) + return (int(match.group(1)) if match else sys.maxsize, path.name) + + +def parse_columns(body: str, custom_types: Dict[str, str]) -> List[Column]: + columns: List[Column] = [] + for part in split_top_level(body, ","): + tokens = part.strip().split() + if len(tokens) < 2: + continue + if tokens[0].upper() in {"PRIMARY", "FOREIGN", "UNIQUE", "CHECK", "CONSTRAINT"}: + continue + + name = clean_identifier(tokens[0]) + sql_type = clean_identifier(tokens[1]).upper() + constraints = " ".join(tokens[2:]).upper() + nullable = "NOT NULL" not in constraints and "PRIMARY KEY" not in constraints + ts_type = custom_types.get(sql_type, default_ts_type(sql_type)) + if nullable and ts_type != "any": + ts_type = f"{ts_type} | null" + columns.append(Column(name=name, sql_type=sql_type, ts_type=ts_type, nullable=nullable)) + return columns + + +def parse_sql_file(path: Path, package_dir: Path, tables: Dict[str, Table]) -> SqlFile: + rel_to_package = path.relative_to(package_dir) + stem_path = rel_to_package.with_suffix("") + query_blocks = parse_query_blocks(path.read_text(encoding="utf-8")) + query_names: set[str] = set() + for name, _ in query_blocks: + if name in query_names: + raise ClientSqlError(f"Duplicate query name '{name}' in {rel_to_package}") + query_names.add(name) + queries = [ + analyze_query(name, sql, tables) + for name, sql in query_blocks + ] + return SqlFile(path=path, rel_to_package=rel_to_package, stem_path=stem_path, queries=queries) + + +def parse_query_blocks(sql_text: str) -> List[Tuple[str, str]]: + blocks: List[Tuple[str, str]] = [] + current_name: Optional[str] = None + current_lines: List[str] = [] + + def flush() -> None: + nonlocal current_name, current_lines + if current_name is not None: + sql = "\n".join(current_lines).strip() + if sql: + blocks.append((current_name, sql)) + current_name = None + current_lines = [] + + for raw_line in sql_text.splitlines(): + label = re.match(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*(.*)$", raw_line) + if label and not raw_line.lstrip().startswith("--"): + flush() + current_name = label.group(1) + rest = label.group(2).strip() + if rest: + current_lines.append(rest) + continue + if current_name is not None: + current_lines.append(raw_line) + + flush() + return blocks + + +def analyze_query(name: str, sql: str, tables: Dict[str, Table]) -> Query: + runtime_sql, param_order = normalize_params(sql) + params = infer_params(sql, param_order, tables) + read_tables = tables_read_by_query(sql, tables) + changed_tables = tables_changed_by_query(sql, tables) + # Result-shape inference is deliberately conservative. SQLite validates more + # statement forms below, but generated row types are only promised for the + # SELECT grammar that infer_result_fields() understands. + returns_rows = bool(re.match(r"^\s*SELECT\b", sql, re.IGNORECASE)) + result_fields = infer_result_fields(sql, tables) if returns_rows else [] + result_type = f"{sanitize_type_name(name)}Row" if returns_rows else "void" + query_table = table_for_query(sql, tables) + if returns_rows: + star_table = select_star_table(sql, tables) + if star_table is not None: + result_type = sanitize_type_name(star_table.name) + if not returns_rows: + read_tables = [] + + return Query( + name=name, + sql=sql.strip(), + runtime_sql=runtime_sql.strip(), + param_order=param_order, + params=params, + result_type=result_type, + result_fields=result_fields, + returns_rows=returns_rows, + read_tables=read_tables, + changed_tables=changed_tables, + ) + + +def normalize_params(sql: str) -> Tuple[str, List[str]]: + out: List[str] = [] + params: List[str] = [] + index = 0 + i = 0 + quote: Optional[str] = None + while i < len(sql): + ch = sql[i] + if quote: + out.append(ch) + if ch == quote: + if i + 1 < len(sql) and sql[i + 1] == quote: + out.append(sql[i + 1]) + i += 2 + continue + quote = None + i += 1 + continue + if ch in {"'", '"'}: + quote = ch + out.append(ch) + i += 1 + continue + if ch == "?": + param_name = f"p{index}" + params.append(param_name) + out.append("?") + index += 1 + i += 1 + continue + if ch == ":" and i + 1 < len(sql) and re.match(r"[A-Za-z_]", sql[i + 1]): + match = re.match(r":([A-Za-z_][A-Za-z0-9_]*)(\?)?", sql[i:]) + if match: + params.append(match.group(1)) + out.append("?") + i += len(match.group(0)) + continue + out.append(ch) + i += 1 + return "".join(out), params + + +def infer_params(sql: str, param_order: List[str], tables: Dict[str, Table]) -> List[Parameter]: + unique_order: List[str] = [] + for name in param_order: + if name not in unique_order: + unique_order.append(name) + + insert_types = infer_insert_param_types(sql, param_order, tables) + update_types = infer_update_param_types(sql, param_order, tables) + predicate_types = infer_predicate_param_types(sql, param_order, tables) + limit_types = infer_limit_param_types(sql, param_order) + nullable_params = infer_nullable_params(sql, param_order) + params: List[Parameter] = [] + for name in unique_order: + ts_type = ( + insert_types.get(name) + or update_types.get(name) + or predicate_types.get(name) + or limit_types.get(name) + or "ClientSQLValue" + ) + if name in nullable_params: + ts_type = nullable_type(ts_type) + params.append(Parameter(name=sanitize_identifier(name), ts_type=ts_type)) + return params + + +def infer_insert_param_types(sql: str, param_order: List[str], tables: Dict[str, Table]) -> Dict[str, str]: + match = re.search( + r"\bINSERT\s+(?:OR\s+\w+\s+)?INTO\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\)\s*VALUES\s*\((.*?)\)", + sql, + re.IGNORECASE | re.DOTALL, + ) + if not match: + return {} + + table = tables.get(match.group(1).lower()) + if table is None: + return {} + + columns = [clean_identifier(part.strip()) for part in split_top_level(match.group(2), ",")] + values = [part.strip() for part in split_top_level(match.group(3), ",")] + result: Dict[str, str] = {} + for idx, value in enumerate(values): + if idx >= len(columns): + continue + param_name: Optional[str] = None + named = re.fullmatch(r":([A-Za-z_][A-Za-z0-9_]*)(?:\?)?", value) + if named: + param_name = named.group(1) + elif value == "?" and idx < len(param_order): + param_name = param_order[idx] + if param_name is None: + continue + column = find_column(table, columns[idx]) + if column: + result[param_name] = column.ts_type + return result + + +def infer_update_param_types(sql: str, param_order: List[str], tables: Dict[str, Table]) -> Dict[str, str]: + match = re.search( + r"\bUPDATE\s+([A-Za-z_][A-Za-z0-9_]*)\s+SET\s+(.*?)(?:\s+WHERE\b|\s+ORDER\s+BY\b|\s+LIMIT\b|;|$)", + sql, + re.IGNORECASE | re.DOTALL, + ) + if not match: + return {} + + table = tables.get(match.group(1).lower()) + if table is None: + return {} + + set_body = match.group(2) + search_start = match.start(2) + result: Dict[str, str] = {} + for assignment in split_top_level(set_body, ","): + assignment_start = sql.find(assignment, search_start) + if assignment_start == -1: + assignment_start = search_start + search_start = assignment_start + len(assignment) + assignment_match = re.match(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$", assignment, re.DOTALL) + if not assignment_match: + continue + column = find_column(table, assignment_match.group(1)) + if column is None: + continue + expr_start = assignment_start + assignment_match.start(2) + expr_end = assignment_start + assignment_match.end(2) + for param_name in param_names_between(sql, expr_start, expr_end, param_order): + result[param_name] = column.ts_type + return result + + +def infer_predicate_param_types(sql: str, param_order: List[str], tables: Dict[str, Table]) -> Dict[str, str]: + table = table_for_query(sql, tables) + if table is None: + return {} + + result: Dict[str, str] = {} + for match in re.finditer( + r"\b([A-Za-z_][A-Za-z0-9_]*)\b\s*(?:<=|>=|!=|<>|=|<|>|LIKE|IN)\s*(\?|:[A-Za-z_][A-Za-z0-9_]*\??)", + sql, + re.IGNORECASE, + ): + column = find_column(table, match.group(1)) + if column is None: + continue + param_name = param_name_at(sql, match.start(2), param_order) + if param_name is not None: + result[param_name] = column.ts_type + return result + + +def infer_limit_param_types(sql: str, param_order: List[str]) -> Dict[str, str]: + result: Dict[str, str] = {} + value_pattern = r"(\?|:[A-Za-z_][A-Za-z0-9_]*\??)" + for match in re.finditer(rf"\bLIMIT\s+{value_pattern}(?:\s*,\s*{value_pattern})?", sql, re.IGNORECASE): + param_name = param_name_at(sql, match.start(1), param_order) + if param_name is not None: + result[param_name] = "number" + if match.lastindex and match.lastindex >= 2 and match.group(2): + param_name = param_name_at(sql, match.start(2), param_order) + if param_name is not None: + result[param_name] = "number" + for match in re.finditer(rf"\bOFFSET\s+{value_pattern}", sql, re.IGNORECASE): + param_name = param_name_at(sql, match.start(1), param_order) + if param_name is not None: + result[param_name] = "number" + return result + + +def infer_nullable_params(sql: str, param_order: List[str]) -> set[str]: + occurrences = scan_param_occurrences(sql) + nullable = {occurrence.name for occurrence in occurrences if occurrence.nullable} + nullable.update( + match.group(1) + for match in re.finditer(r":([A-Za-z_][A-Za-z0-9_]*)\s+IS\s+(?:NOT\s+)?NULL\b", sql, re.IGNORECASE) + ) + + for match in re.finditer(r"\?\s+IS\s+(?:NOT\s+)?NULL\b", sql, re.IGNORECASE): + param_name = param_name_at(sql, match.start(), param_order) + if param_name is not None: + nullable.add(param_name) + return nullable + + +def param_names_between(sql: str, start: int, end: int, param_order: List[str]) -> List[str]: + del param_order + return [ + occurrence.name + for occurrence in scan_param_occurrences(sql) + if start <= occurrence.start and occurrence.end <= end + ] + + +def param_name_at(sql: str, start: int, param_order: List[str]) -> Optional[str]: + del param_order + for occurrence in scan_param_occurrences(sql): + if occurrence.start == start: + return occurrence.name + return None + + +def scan_param_occurrences(sql: str) -> List[ParamOccurrence]: + occurrences: List[ParamOccurrence] = [] + positional_index = 0 + i = 0 + quote: Optional[str] = None + while i < len(sql): + ch = sql[i] + if quote: + if ch == quote: + if i + 1 < len(sql) and sql[i + 1] == quote: + i += 2 + continue + quote = None + i += 1 + continue + if ch in {"'", '"'}: + quote = ch + i += 1 + continue + if ch == "?": + occurrences.append(ParamOccurrence(start=i, end=i + 1, name=f"p{positional_index}", nullable=False)) + positional_index += 1 + i += 1 + continue + if ch == ":" and i + 1 < len(sql) and re.match(r"[A-Za-z_]", sql[i + 1]): + match = re.match(r":([A-Za-z_][A-Za-z0-9_]*)(\?)?", sql[i:]) + if match: + occurrences.append( + ParamOccurrence( + start=i, + end=i + len(match.group(0)), + name=match.group(1), + nullable=bool(match.group(2)), + ) + ) + i += len(match.group(0)) + continue + i += 1 + return occurrences + + +def infer_result_fields(sql: str, tables: Dict[str, Table]) -> List[Column]: + table = table_for_query(sql, tables) + if table is None: + return [] + + select_match = re.search(r"\bSELECT\s+(.*?)\s+FROM\b", sql, re.IGNORECASE | re.DOTALL) + if not select_match: + return [] + + selected = select_match.group(1).strip() + if selected == "*": + return table.columns + + fields: List[Column] = [] + for part in split_top_level(selected, ","): + expression = part.strip() + alias_match = re.search(r"\s+AS\s+([A-Za-z_][A-Za-z0-9_]*)$", expression, re.IGNORECASE) + if alias_match: + name = alias_match.group(1) + source = expression[: alias_match.start()].strip() + else: + source = expression + name = clean_identifier(source.split(".")[-1].strip()) + + column = find_column(table, source.split(".")[-1].strip()) + if column: + fields.append(Column(name=name, sql_type=column.sql_type, ts_type=column.ts_type, nullable=column.nullable)) + else: + fields.append(Column(name=name, sql_type="ANY", ts_type=infer_expression_type(source), nullable=True)) + return fields + + +def select_star_table(sql: str, tables: Dict[str, Table]) -> Optional[Table]: + if not re.search(r"\bSELECT\s+\*\s+FROM\b", sql, re.IGNORECASE | re.DOTALL): + return None + return table_for_query(sql, tables) + + +def table_for_query(sql: str, tables: Dict[str, Table]) -> Optional[Table]: + for pattern in [ + r"\bFROM\s+([A-Za-z_][A-Za-z0-9_]*)", + r"\bUPDATE\s+([A-Za-z_][A-Za-z0-9_]*)", + r"\bINSERT\s+(?:OR\s+\w+\s+)?INTO\s+([A-Za-z_][A-Za-z0-9_]*)", + r"\bDELETE\s+FROM\s+([A-Za-z_][A-Za-z0-9_]*)", + ]: + match = re.search(pattern, sql, re.IGNORECASE) + if match: + return tables.get(match.group(1).lower()) + return None + + +def tables_read_by_query(sql: str, tables: Dict[str, Table]) -> List[str]: + names: List[str] = [] + pattern = rf"\b(?:FROM|JOIN)\s+({SQL_IDENTIFIER_PATTERN})" + for match in re.finditer(pattern, sql, re.IGNORECASE): + table = tables.get(clean_identifier(match.group(1)).lower()) + if table is not None and table.name not in names: + names.append(table.name) + return names + + +def tables_changed_by_query(sql: str, tables: Dict[str, Table]) -> List[str]: + names: List[str] = [] + patterns = [ + rf"\bUPDATE\s+({SQL_IDENTIFIER_PATTERN})", + rf"\b(?:INSERT|REPLACE)\s+(?:OR\s+\w+\s+)?INTO\s+({SQL_IDENTIFIER_PATTERN})", + rf"\bDELETE\s+FROM\s+({SQL_IDENTIFIER_PATTERN})", + ] + for pattern in patterns: + for match in re.finditer(pattern, sql, re.IGNORECASE): + table = tables.get(clean_identifier(match.group(1)).lower()) + if table is not None and table.name not in names: + names.append(table.name) + return names + + +def find_column(table: Table, name: str) -> Optional[Column]: + cleaned = clean_identifier(name).lower() + for column in table.columns: + if column.name.lower() == cleaned: + return column + return None + + +def infer_expression_type(expression: str) -> str: + if re.search(r"\bCOUNT\s*\(", expression, re.IGNORECASE): + return "number" + if re.search(r"\b(?:SUM|AVG|MIN|MAX)\s*\(", expression, re.IGNORECASE): + return "number | null" + return "any" + + +def default_ts_type(sql_type: str) -> str: + normalized = sql_type.upper() + if normalized in {"INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT"}: + return "number" + if normalized in {"REAL", "DOUBLE", "FLOAT", "NUMERIC", "DECIMAL"}: + return "number" + if normalized in {"TEXT", "CHAR", "CLOB", "VARCHAR", "NCHAR", "NVARCHAR"}: + return "string" + if normalized == "BLOB": + return "ArrayBuffer" + if normalized in {"BOOL", "BOOLEAN"}: + return "boolean" + return "any" + + +def nullable_type(ts_type: str) -> str: + if ts_type == "any" or re.search(r"(?:^|\|\s*)null(?:\s*\||$)", ts_type): + return ts_type + return f"{ts_type} | null" + + +def split_sql_statements(sql_text: str) -> List[str]: + statements: List[str] = [] + current: List[str] = [] + quote: Optional[str] = None + i = 0 + while i < len(sql_text): + ch = sql_text[i] + current.append(ch) + if quote: + if ch == quote: + if i + 1 < len(sql_text) and sql_text[i + 1] == quote: + current.append(sql_text[i + 1]) + i += 2 + continue + quote = None + elif ch in {"'", '"'}: + quote = ch + elif ch == ";": + statement = "".join(current).strip() + if statement: + statements.append(statement) + current = [] + i += 1 + + tail = "".join(current).strip() + if tail: + statements.append(tail) + return statements + + +def split_top_level(text: str, delimiter: str) -> List[str]: + parts: List[str] = [] + current: List[str] = [] + depth = 0 + quote: Optional[str] = None + for ch in text: + if quote: + current.append(ch) + if ch == quote: + quote = None + continue + if ch in {"'", '"'}: + quote = ch + current.append(ch) + continue + if ch == "(": + depth += 1 + elif ch == ")": + depth = max(0, depth - 1) + if ch == delimiter and depth == 0: + parts.append("".join(current)) + current = [] + else: + current.append(ch) + parts.append("".join(current)) + return parts + + +def strip_sql_comments(sql_text: str) -> str: + without_block = re.sub(r"/\*.*?\*/", "", sql_text, flags=re.DOTALL) + return "\n".join(line.split("--", 1)[0] for line in without_block.splitlines()) + + +def clean_identifier(identifier: str) -> str: + return identifier.strip().strip("`\"[]") + + +def sanitize_identifier(name: str) -> str: + cleaned = re.sub(r"\W+", "_", name) + if not cleaned or re.match(r"\d", cleaned): + cleaned = f"p_{cleaned}" + return cleaned + + +def sanitize_type_name(name: str) -> str: + parts = re.split(r"[^A-Za-z0-9]+", name) + result = "".join(part[:1].upper() + part[1:] for part in parts if part) + if not result: + return "Generated" + if re.match(r"\d", result): + return f"T{result}" + return result diff --git a/compiler/clientsql/src/clientsql/typescript.py b/compiler/clientsql/src/clientsql/typescript.py new file mode 100644 index 00000000..4ae878a4 --- /dev/null +++ b/compiler/clientsql/src/clientsql/typescript.py @@ -0,0 +1,726 @@ +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Dict, List, Sequence, Tuple + +from .model import Query, SqlFile, Table +from .sql import sanitize_identifier, sanitize_type_name + + +def write_types_file(output_dir: Path, sql_file: SqlFile, tables: Dict[str, Table]) -> None: + path = output_dir / f"{sql_file.stem_path}Types.ts" + path.parent.mkdir(parents=True, exist_ok=True) + + content = render_types(sql_file, tables, declare=False) + path.write_text(content, encoding="utf-8") + + +def render_types(sql_file: SqlFile, tables: Dict[str, Table], declare: bool) -> str: + lines = generated_header() + used_tables = tables_used_by_file(sql_file, tables) + for table in used_tables: + lines.extend(render_interface(sanitize_type_name(table.name), table.columns, declare_export=True)) + lines.append("") + + for query in sql_file.queries: + if query.params: + lines.extend(render_interface(f"{sanitize_type_name(query.name)}Params", query.params, declare_export=True)) + lines.append("") + if query.returns_rows and query.result_type == f"{sanitize_type_name(query.name)}Row": + lines.extend(render_interface(query.result_type, query.result_fields, declare_export=True)) + lines.append("") + + if len(lines) == len(generated_header()): + lines.append("export {};") + return "\n".join(lines).rstrip() + "\n" + + +def write_queries_file(output_dir: Path, sql_file: SqlFile) -> None: + path = output_dir / f"{sql_file.stem_path}Queries.ts" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_queries(sql_file, declare=False), encoding="utf-8") + + +def render_queries(sql_file: SqlFile, declare: bool) -> str: + class_name = f"{sanitize_type_name(sql_file.stem_path.name)}Queries" + type_imports = sorted(types_imported_by_queries(sql_file.queries)) + import_path = f"./{sql_file.stem_path.name}Types" + + lines = generated_header() + if type_imports: + lines.append(f"import {{ {', '.join(type_imports)} }} from '{import_path}';") + lines.append("") + + lines.extend([ + "export type ClientSQLValue = string | number | boolean | ArrayBuffer | null;", + "export interface ClientSQLSubscription {", + " unsubscribe(): void;", + "}", + "export type ClientSQLQueryListener = (value: T) => void;", + "", + "export interface ClientSQLDatabase {", + " execute(sql: string, parameters?: ClientSQLValue[], changedTables?: string[]): Promise;", + " query(sql: string, parameters?: ClientSQLValue[]): Promise;", + " watchQuery(", + " tables: string[],", + " load: () => Promise,", + " listener: ClientSQLQueryListener,", + " ): ClientSQLSubscription;", + "}", + "", + ]) + + if declare: + lines.append(f"export declare class {class_name} {{") + lines.append(" constructor(db: ClientSQLDatabase);") + for query in sql_file.queries: + lines.append(f" {query.name}({method_signature_params(query)}): {method_return_type(query)};") + if query.returns_rows: + lines.append(f" {watch_method_name(query)}({watch_method_signature_params(query)}): ClientSQLSubscription;") + lines.append("}") + return "\n".join(lines).rstrip() + "\n" + + lines.append(f"export class {class_name} {{") + lines.append(" constructor(private readonly db: ClientSQLDatabase) {}") + lines.append("") + for query in sql_file.queries: + lines.extend(render_query_method(query)) + if query.returns_rows: + lines.extend(render_query_watch_method(query)) + lines.append("") + lines.append("}") + return "\n".join(lines).rstrip() + "\n" + + +def render_query_method(query: Query) -> List[str]: + sql_literal = json.dumps(query.runtime_sql) + param_array = ", ".join(sanitize_identifier(name) for name in query.param_order) + lines = [ + f" {query.name}({method_signature_params(query)}): {method_return_type(query)} {{", + ] + if query.returns_rows: + lines.append(f" return this.db.query<{query.result_type}>({sql_literal}, [{param_array}]);") + elif query.changed_tables: + lines.append(f" return this.db.execute({sql_literal}, [{param_array}], {json.dumps(query.changed_tables)});") + else: + lines.append(f" return this.db.execute({sql_literal}, [{param_array}]);") + lines.append(" }") + return lines + + +def render_query_watch_method(query: Query) -> List[str]: + tables_literal = json.dumps(query.read_tables) + param_values = ", ".join(param.name for param in query.params) + invocation = f"this.{query.name}({param_values})" if param_values else f"this.{query.name}()" + return [ + f" {watch_method_name(query)}({watch_method_signature_params(query)}): ClientSQLSubscription {{", + f" return this.db.watchQuery({tables_literal}, () => {invocation}, listener);", + " }", + ] + + +def write_database_file( + output_dir: Path, + class_name: str, + db_name: str, + sql_files: List[SqlFile], + create_statements: List[str], + migrations: List[Tuple[int, List[str]]], +) -> None: + path = output_dir / f"{class_name}.ts" + path.write_text(render_database(class_name, db_name, sql_files, create_statements, migrations, declare=False), encoding="utf-8") + + +def render_database( + class_name: str, + db_name: str, + sql_files: List[SqlFile], + create_statements: List[str], + migrations: List[Tuple[int, List[str]]], + declare: bool, +) -> str: + query_classes = [(query_class_name(sql_file), import_path_from_db(sql_file, "Queries")) for sql_file in sql_files if sql_file.queries] + all_tables = sorted({ + table + for sql_file in sql_files + for query in sql_file.queries + for table in query.read_tables + query.changed_tables + }) + lines = generated_header() + for klass, import_path in query_classes: + lines.append(f"import {{ {klass} }} from '{import_path}';") + if not declare: + lines.append("import { ClientSQLDebugDatabase, notifyClientSQLDebugChanged, registerClientSQLDebugDatabase, unregisterClientSQLDebugDatabase } from 'client_sql/src/ClientSQLDebug';") + if query_classes or not declare: + lines.append("") + + lines.extend([ + "declare function require(path: string): any;", + "", + "export type ClientSQLValue = string | number | boolean | ArrayBuffer | null;", + "export type ClientSQLNativeCallback = (value: T | undefined, error: string | undefined) => void;", + "", + "export interface ClientSQLSubscription {", + " unsubscribe(): void;", + "}", + "", + "interface ClientSQLTransactionDebugEntry {", + " id: number;", + " status: 'committed' | 'rolled_back';", + " startedAt: string;", + " completedAt: string;", + " durationMs: number;", + " changedTables: string[];", + " changedTableCount: number;", + " error?: string;", + "}", + "", + "const MAX_TRANSACTION_DEBUG_ENTRIES = 50;", + "", + "export type ClientSQLQueryListener = (value: T) => void;", + "", + "export interface ClientSQLMigration {", + " version: number;", + " statements: string[];", + "}", + "", + "export interface ClientSQLNativeTransaction {", + " execute(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void;", + " query(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void;", + "}", + "", + "export type ClientSQLNativeTransactionBody = (", + " transaction: ClientSQLNativeTransaction,", + " callback: ClientSQLNativeCallback,", + ") => void;", + "", + "export interface ClientSQLNativeConnection {", + " execute(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void;", + " query(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void;", + " transaction(body: ClientSQLNativeTransactionBody, callback: ClientSQLNativeCallback): void;", + " close(callback: ClientSQLNativeCallback): void;", + "}", + "", + "export interface ClientSQLNativeModule {", + " openDatabase(", + " name: string,", + " schemaVersion: number,", + " createStatements: string[],", + " migrations: ClientSQLMigration[],", + " ): ClientSQLNativeConnection;", + "}", + "", + ]) + + lines.append(f"export interface {class_name}Transaction {{") + for klass, _ in query_classes: + property_name = lower_first(klass.removesuffix("Queries")) + "Queries" + lines.append(f" readonly {property_name}: {klass};") + lines.extend([ + " execute(sql: string, parameters?: ClientSQLValue[], changedTables?: string[]): Promise;", + " query(sql: string, parameters?: ClientSQLValue[]): Promise;", + f" transaction(body: (transaction: {class_name}Transaction) => Promise): Promise;", + "}", + "", + ]) + + if declare: + lines.append("export declare function setClientSQLNativeForTests(native: ClientSQLNativeModule | undefined): void;") + lines.append("") + lines.append(f"export declare class {class_name} {{") + for klass, _ in query_classes: + property_name = lower_first(klass.removesuffix("Queries")) + "Queries" + lines.append(f" readonly {property_name}: {klass};") + lines.append(" static open(name?: string): " + class_name + ";") + lines.append(" execute(sql: string, parameters?: ClientSQLValue[], changedTables?: string[]): Promise;") + lines.append(" query(sql: string, parameters?: ClientSQLValue[]): Promise;") + lines.append(" watchQuery(") + lines.append(" tables: string[],") + lines.append(" load: () => Promise,") + lines.append(" listener: ClientSQLQueryListener,") + lines.append(" ): ClientSQLSubscription;") + lines.append( + f" transaction(body: (transaction: {class_name}Transaction) => Promise): Promise;" + ) + lines.append(" close(): Promise;") + lines.append("}") + return "\n".join(lines).rstrip() + "\n" + + schema_version = max([version for version, _ in migrations], default=1) + lines.extend([ + f"const DEFAULT_DATABASE_NAME = {json.dumps(db_name)};", + f"const SCHEMA_VERSION = {schema_version};", + f"const CREATE_STATEMENTS: string[] = {json.dumps(create_statements, indent=2)};", + f"const MIGRATIONS: ClientSQLMigration[] = {json.dumps([{'version': version, 'statements': statements} for version, statements in migrations], indent=2)};", + f"const ALL_TABLES: string[] = {json.dumps(all_tables)};", + "", + "let clientSQLNativeForTests: ClientSQLNativeModule | undefined;", + "", + "interface ClientSQLWatchEntry {", + " tables: string[];", + " emit(): void;", + " unsubscribe(): void;", + "}", + "", + "const watchEntriesByDatabaseName: { [name: string]: ClientSQLWatchEntry[] | undefined } = Object.create(null);", + "const writeChainsByDatabaseName: { [name: string]: Promise | undefined } = Object.create(null);", + "", + "export function setClientSQLNativeForTests(native: ClientSQLNativeModule | undefined): void {", + " clientSQLNativeForTests = native;", + "}", + "", + "function getClientSQLNative(): ClientSQLNativeModule {", + " return clientSQLNativeForTests ?? (require('client_sql/src/ClientSQLNative') as ClientSQLNativeModule);", + "}", + "", + "function entriesForDatabase(name: string): ClientSQLWatchEntry[] {", + " let entries = watchEntriesByDatabaseName[name];", + " if (!entries) {", + " entries = [];", + " watchEntriesByDatabaseName[name] = entries;", + " }", + " return entries;", + "}", + "", + "function removeWatchEntry(entries: ClientSQLWatchEntry[], entry: ClientSQLWatchEntry): void {", + " const index = entries.indexOf(entry);", + " if (index !== -1) {", + " entries.splice(index, 1);", + " }", + "}", + "", + "function releaseDatabaseWrite(name: string, chain: Promise, release: () => void): void {", + " release();", + " if (writeChainsByDatabaseName[name] === chain) {", + " delete writeChainsByDatabaseName[name];", + " }", + "}", + "", + "function enqueueDatabaseWrite(name: string, body: () => Promise): Promise {", + " const previous = writeChainsByDatabaseName[name] ?? Promise.resolve();", + " let releaseCurrent!: () => void;", + " const current = new Promise(resolve => {", + " releaseCurrent = resolve;", + " });", + " const chain = previous.then(() => current, () => current);", + " writeChainsByDatabaseName[name] = chain;", + "", + " const runBody = (): Promise => {", + " let result: Promise;", + " try {", + " result = body();", + " } catch (error) {", + " releaseDatabaseWrite(name, chain, releaseCurrent);", + " return Promise.reject(error);", + " }", + "", + " return result.then(", + " value => {", + " releaseDatabaseWrite(name, chain, releaseCurrent);", + " return value;", + " },", + " error => {", + " releaseDatabaseWrite(name, chain, releaseCurrent);", + " throw error;", + " },", + " );", + " };", + "", + " return previous.then(runBody, runBody);", + "}", + "", + "function toUniqueTables(tables: string[]): string[] {", + " const out: string[] = [];", + " tables.forEach(table => {", + " if (out.indexOf(table) === -1) {", + " out.push(table);", + " }", + " });", + " return out;", + "}", + "", + "function hasTableIntersection(observedTables: string[], changedTables: string[]): boolean {", + " if (observedTables.length === 0 || changedTables.length === 0) {", + " return true;", + " }", + " return observedTables.some(table => changedTables.indexOf(table) !== -1);", + "}", + "", + "function nativePromise(body: (callback: ClientSQLNativeCallback) => void): Promise {", + " return new Promise((resolve, reject) => {", + " body((value, error) => {", + " if (error !== undefined && error !== null) {", + " reject(new Error(error));", + " return;", + " }", + " resolve(value as T);", + " });", + " });", + "}", + "", + "function errorMessage(error: unknown): string {", + " return error instanceof Error ? error.message : String(error);", + "}", + "", + f"export class {class_name} {{", + " private readonly localWatchEntries: ClientSQLWatchEntry[] = [];", + " private debugDatabase: ClientSQLDebugDatabase | undefined;", + " private activeTransactionCount = 0;", + " private activeTransactionChangedTables: string[] | undefined;", + " private nextTransactionDebugId = 1;", + " private transactionHistory: ClientSQLTransactionDebugEntry[] = [];", + " private closed = false;", + "", + " private constructor(", + " private readonly databaseName: string,", + " private readonly connection: ClientSQLNativeConnection,", + " ) {", + ]) + for klass, _ in query_classes: + property_name = lower_first(klass.removesuffix("Queries")) + "Queries" + lines.append(f" this.{property_name} = new {klass}(this);") + lines.extend([ + " this.debugDatabase = {", + " name: this.databaseName,", + " schemaVersion: SCHEMA_VERSION,", + " createStatements: CREATE_STATEMENTS,", + " migrations: MIGRATIONS,", + " query: (sql: string, parameters: ClientSQLValue[] = []): Promise => this.query(sql, parameters),", + " debugInfo: (): Promise> => this.debugInfo(),", + " };", + " registerClientSQLDebugDatabase(this.debugDatabase);", + ]) + lines.append(" }") + lines.append("") + for klass, _ in query_classes: + property_name = lower_first(klass.removesuffix("Queries")) + "Queries" + lines.append(f" readonly {property_name}: {klass};") + if query_classes: + lines.append("") + lines.extend([ + f" static open(name: string = DEFAULT_DATABASE_NAME): {class_name} {{", + " const connection = getClientSQLNative().openDatabase(name, SCHEMA_VERSION, CREATE_STATEMENTS, MIGRATIONS);", + f" return new {class_name}(name, connection);", + " }", + "", + " async execute(sql: string, parameters: ClientSQLValue[] = [], changedTables?: string[]): Promise {", + " if (this.closed) {", + " throw new Error(`ClientSQL database '${this.databaseName}' is closed`);", + " }", + " return enqueueDatabaseWrite(this.databaseName, async () => {", + " await nativePromise(callback => this.connection.execute(sql, parameters, callback));", + " this.notifyTablesChanged(changedTables ?? ALL_TABLES);", + " });", + " }", + "", + " async query(sql: string, parameters: ClientSQLValue[] = []): Promise {", + " if (this.closed) {", + " throw new Error(`ClientSQL database '${this.databaseName}' is closed`);", + " }", + " return nativePromise(callback => this.connection.query(sql, parameters, callback));", + " }", + "", + " watchQuery(", + " tables: string[],", + " load: () => Promise,", + " listener: ClientSQLQueryListener,", + " ): ClientSQLSubscription {", + " if (this.closed) {", + " return { unsubscribe(): void {} };", + " }", + "", + " let active = true;", + " let generation = 0;", + " const emit = (): void => {", + " const currentGeneration = ++generation;", + " void load()", + " .then(value => {", + " if (active && currentGeneration === generation) {", + " listener(value);", + " }", + " })", + " .catch(error => {", + " if (active) {", + " console.error(error);", + " }", + " });", + " };", + "", + " const unsubscribe = (): void => {", + " if (!active) {", + " return;", + " }", + " active = false;", + " const entries = watchEntriesByDatabaseName[this.databaseName];", + " if (entries) {", + " removeWatchEntry(entries, entry);", + " }", + " removeWatchEntry(this.localWatchEntries, entry);", + " };", + " const entry: ClientSQLWatchEntry = { tables, emit, unsubscribe };", + " entriesForDatabase(this.databaseName).push(entry);", + " this.localWatchEntries.push(entry);", + " emit();", + "", + " return {", + " unsubscribe,", + " };", + " }", + "", + f" async transaction(body: (transaction: {class_name}Transaction) => Promise): Promise {{", + " if (this.closed) {", + " throw new Error(`ClientSQL database '${this.databaseName}' is closed`);", + " }", + " return enqueueDatabaseWrite(this.databaseName, () => this.runTransaction(body));", + " }", + "", + f" private async runTransaction(body: (transaction: {class_name}Transaction) => Promise): Promise {{", + " let result!: T;", + " const changedTables: string[] = [];", + " const transactionDebugId = this.nextTransactionDebugId++;", + " const transactionStartedAtMs = Date.now();", + " this.activeTransactionCount += 1;", + " this.activeTransactionChangedTables = changedTables;", + "", + " try {", + " await nativePromise(callback => {", + " this.connection.transaction((nativeTransaction, transactionCallback) => {", + " const transaction = this.createTransactionScope(nativeTransaction, changedTables);", + " let bodyPromise: Promise;", + " try {", + " bodyPromise = body(transaction);", + " } catch (error) {", + " transactionCallback(undefined, errorMessage(error));", + " return;", + " }", + "", + " void bodyPromise.then(", + " value => {", + " result = value;", + " transactionCallback(undefined, undefined);", + " },", + " error => {", + " transactionCallback(undefined, errorMessage(error));", + " },", + " );", + " }, callback);", + " });", + " const transactionCompletedAtMs = Date.now();", + " this.recordTransactionDebugEntry({", + " id: transactionDebugId,", + " status: 'committed',", + " startedAt: new Date(transactionStartedAtMs).toISOString(),", + " completedAt: new Date(transactionCompletedAtMs).toISOString(),", + " durationMs: transactionCompletedAtMs - transactionStartedAtMs,", + " changedTables: changedTables.slice(),", + " changedTableCount: changedTables.length,", + " });", + " this.emitChangedTables(changedTables);", + " return result;", + " } catch (error) {", + " const transactionCompletedAtMs = Date.now();", + " this.recordTransactionDebugEntry({", + " id: transactionDebugId,", + " status: 'rolled_back',", + " startedAt: new Date(transactionStartedAtMs).toISOString(),", + " completedAt: new Date(transactionCompletedAtMs).toISOString(),", + " durationMs: transactionCompletedAtMs - transactionStartedAtMs,", + " changedTables: changedTables.slice(),", + " changedTableCount: changedTables.length,", + " error: errorMessage(error),", + " });", + " notifyClientSQLDebugChanged(this.databaseName);", + " throw error;", + " } finally {", + " this.activeTransactionChangedTables = undefined;", + " this.activeTransactionCount -= 1;", + " }", + " }", + "", + f" private createTransactionScope(nativeTransaction: ClientSQLNativeTransaction, changedTables: string[]): {class_name}Transaction {{", + " const transactionDatabase = {", + " execute: async (sql: string, parameters: ClientSQLValue[] = [], tables?: string[]): Promise => {", + " await nativePromise(callback => nativeTransaction.execute(sql, parameters, callback));", + " const invalidatedTables = tables ?? ALL_TABLES;", + " invalidatedTables.forEach(table => {", + " if (changedTables.indexOf(table) === -1) {", + " changedTables.push(table);", + " }", + " });", + " },", + " query: (sql: string, parameters: ClientSQLValue[] = []): Promise =>", + " nativePromise(callback => nativeTransaction.query(sql, parameters, callback)),", + " watchQuery: (", + " _tables: string[],", + " _load: () => Promise,", + " _listener: ClientSQLQueryListener,", + " ): ClientSQLSubscription => {", + " throw new Error('ClientSQL watchers cannot be created inside a transaction');", + " },", + " };", + f" let scope!: {class_name}Transaction;", + " scope = {", + ]) + for klass, _ in query_classes: + property_name = lower_first(klass.removesuffix("Queries")) + "Queries" + lines.append(f" {property_name}: new {klass}(transactionDatabase),") + lines.extend([ + " execute: transactionDatabase.execute,", + " query: transactionDatabase.query,", + " transaction: nestedBody => nestedBody(scope),", + " };", + " return scope;", + " }", + "", + " async close(): Promise {", + " if (this.closed) {", + " return;", + " }", + " this.closed = true;", + " this.localWatchEntries.slice().forEach(entry => {", + " if (typeof entry.unsubscribe === 'function') {", + " entry.unsubscribe();", + " return;", + " }", + " const entries = watchEntriesByDatabaseName[this.databaseName];", + " if (entries) {", + " removeWatchEntry(entries, entry);", + " }", + " removeWatchEntry(this.localWatchEntries, entry);", + " });", + " if (this.debugDatabase) {", + " unregisterClientSQLDebugDatabase(this.debugDatabase);", + " this.debugDatabase = undefined;", + " }", + " await enqueueDatabaseWrite(this.databaseName, () => nativePromise(callback => this.connection.close(callback)));", + " }", + "", + " private async debugInfo(): Promise> {", + " const debugConnection = this.connection as ClientSQLNativeConnection & {", + " debugInfo?: (callback: ClientSQLNativeCallback>) => void;", + " };", + " const nativeInfo = debugConnection.debugInfo", + " ? await nativePromise>(callback => debugConnection.debugInfo!(callback))", + " : {};", + " return {", + " ...nativeInfo,", + " closed: this.closed,", + " transactionDepth: this.activeTransactionCount,", + " pendingChangedTables: this.activeTransactionChangedTables?.slice() ?? [],", + " pendingChangedTableCount: this.activeTransactionChangedTables?.length ?? 0,", + " transactionHistoryCount: this.transactionHistory.length,", + " transactions: this.transactionHistory.slice().reverse(),", + " watcherCount: (watchEntriesByDatabaseName[this.databaseName] || []).length,", + " localWatcherCount: this.localWatchEntries.length,", + " queuedWrite: writeChainsByDatabaseName[this.databaseName] !== undefined,", + " };", + " }", + "", + " private recordTransactionDebugEntry(entry: ClientSQLTransactionDebugEntry): void {", + " this.transactionHistory.push(entry);", + " const overflow = this.transactionHistory.length - MAX_TRANSACTION_DEBUG_ENTRIES;", + " if (overflow > 0) {", + " this.transactionHistory.splice(0, overflow);", + " }", + " }", + "", + " private notifyTablesChanged(changedTables: string[]): void {", + " const uniqueTables = toUniqueTables(changedTables);", + " this.emitChangedTables(uniqueTables);", + " }", + "", + " private emitChangedTables(changedTables: string[]): void {", + " notifyClientSQLDebugChanged(this.databaseName);", + " const entries = watchEntriesByDatabaseName[this.databaseName];", + " if (!entries) {", + " return;", + " }", + " entries.slice().forEach(entry => {", + " if (hasTableIntersection(entry.tables, changedTables)) {", + " entry.emit();", + " }", + " });", + " }", + "}", + ]) + return "\n".join(lines).rstrip() + "\n" + + +def tables_used_by_file(sql_file: SqlFile, tables: Dict[str, Table]) -> List[Table]: + used: Dict[str, Table] = {} + for query in sql_file.queries: + for table_name in query.read_tables + query.changed_tables: + table = tables.get(table_name.lower()) + if table: + used[table.name.lower()] = table + return sorted(used.values(), key=lambda table: table.name) + + +def types_imported_by_queries(queries: List[Query]) -> List[str]: + imports = set() + for query in queries: + if query.returns_rows: + imports.add(query.result_type) + return sorted(imports) + + +def render_interface(name: str, fields: Sequence[object], declare_export: bool) -> List[str]: + prefix = "export interface" if declare_export else "interface" + lines = [f"{prefix} {name} {{"] + for field in fields: + field_name = getattr(field, "name") + field_type = getattr(field, "ts_type") + lines.append(f" {render_property_name(field_name)}: {field_type};") + lines.append("}") + return lines + + +def method_signature_params(query: Query) -> str: + return ", ".join(f"{param.name}: {param.ts_type}" for param in query.params) + + +def method_return_type(query: Query) -> str: + if query.returns_rows: + return f"Promise<{query.result_type}[]>" + return "Promise" + + +def watch_method_name(query: Query) -> str: + return f"watch{sanitize_type_name(query.name)}" + + +def watch_method_signature_params(query: Query) -> str: + params = method_signature_params(query) + listener = f"listener: ClientSQLQueryListener<{query.result_type}[]>" + if params: + return f"{params}, {listener}" + return listener + + +def query_class_name(sql_file: SqlFile) -> str: + return f"{sanitize_type_name(sql_file.stem_path.name)}Queries" + + +def import_path_from_db(sql_file: SqlFile, suffix: str) -> str: + stem = str(sql_file.stem_path).replace(os.sep, "/") + return f"./{stem}{suffix}" + + +def generated_header() -> List[str]: + return [ + "// Generated by clientsql. Do not edit.", + "", + ] + + +def lower_first(name: str) -> str: + return name[:1].lower() + name[1:] + + +def render_property_name(name: str) -> str: + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + return name + return json.dumps(name) diff --git a/compiler/clientsql/src/clientsql_main.py b/compiler/clientsql/src/clientsql_main.py new file mode 100644 index 00000000..44781cd6 --- /dev/null +++ b/compiler/clientsql/src/clientsql_main.py @@ -0,0 +1,4 @@ +from clientsql.cli import entrypoint + + +entrypoint() diff --git a/compiler/clientsql/test_clientsql.py b/compiler/clientsql/test_clientsql.py new file mode 100644 index 00000000..f0d44664 --- /dev/null +++ b/compiler/clientsql/test_clientsql.py @@ -0,0 +1,947 @@ +import os +import subprocess +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from typing import List + + +def repository_root() -> Path: + test_srcdir = os.environ.get("TEST_SRCDIR") + test_workspace = os.environ.get("TEST_WORKSPACE") + if test_srcdir and test_workspace: + return Path(test_srcdir) / test_workspace + return Path(__file__).resolve().parents[2] + + +def environment_tool_path(name: str) -> Path | None: + value = os.environ.get(name) + if value is None: + return None + path = Path(value) + return path if path.is_absolute() else REPO_ROOT / path + + +REPO_ROOT = repository_root() +CLIENTSQL_EXECUTABLE = environment_tool_path("CLIENTSQL_TEST_GENERATOR") +CLIENTSQL_SOURCE = REPO_ROOT / "compiler" / "clientsql" / "src" / "clientsql_main.py" +CLIENTSQL_PACKAGER = REPO_ROOT / "compiler" / "clientsql" / "package_clientsql.py" +JAVASCRIPT_RUNNER = environment_tool_path("CLIENTSQL_TEST_JAVASCRIPT_RUNNER") +TYPESCRIPT_COMPILER = environment_tool_path("CLIENTSQL_TEST_TYPESCRIPT_COMPILER") +LOCAL_TYPESCRIPT_COMPILER = ( + REPO_ROOT / "npm_modules" / "cli" / "node_modules" / "typescript" / "bin" / "tsc" +) + + +class ClientSQLGeneratorTest(unittest.TestCase): + @staticmethod + def clientsql_command() -> List[str]: + if CLIENTSQL_EXECUTABLE is not None: + if not CLIENTSQL_EXECUTABLE.is_file() or not os.access(CLIENTSQL_EXECUTABLE, os.X_OK): + raise RuntimeError(f"ClientSQL generator is not executable: {CLIENTSQL_EXECUTABLE}") + return [str(CLIENTSQL_EXECUTABLE)] + return [sys.executable, str(CLIENTSQL_SOURCE)] + + def test_packaged_generator_matches_canonical_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + executable = Path(directory) / "clientsql" + package_result = subprocess.run( + [sys.executable, str(CLIENTSQL_PACKAGER), "--output", str(executable)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(package_result.returncode, 0, msg=package_result.stderr) + + check_result = subprocess.run( + [sys.executable, str(CLIENTSQL_PACKAGER), "--output", str(executable), "--check"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(check_result.returncode, 0, msg=check_result.stderr) + + def assert_in_order(self, text: str, *needles: str) -> None: + cursor = 0 + for needle in needles: + index = text.find(needle, cursor) + self.assertNotEqual(index, -1, msg=f"Missing {needle!r} after offset {cursor}") + cursor = index + len(needle) + + def run_clientsql(self, sql_dir: Path, out_dir: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + *self.clientsql_command(), + "-s", + str(sql_dir), + "-p", + "TestDb", + "-c", + "TestDb", + "-m", + "TestDb", + "-o", + str(out_dir), + "-l", + "typescript", + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def node_command(self) -> List[str]: + node = shutil.which("node") + if node: + return [node] + raise RuntimeError("ClientSQL generated-code tests require Node.js") + + def typescript_compiler_command(self) -> List[str]: + if TYPESCRIPT_COMPILER is not None: + if not TYPESCRIPT_COMPILER.is_file() or not os.access(TYPESCRIPT_COMPILER, os.X_OK): + raise RuntimeError(f"TypeScript compiler is not executable: {TYPESCRIPT_COMPILER}") + return [str(TYPESCRIPT_COMPILER)] + if not LOCAL_TYPESCRIPT_COMPILER.is_file(): + raise RuntimeError(f"TypeScript compiler is missing: {LOCAL_TYPESCRIPT_COMPILER}") + return [*self.node_command(), str(LOCAL_TYPESCRIPT_COMPILER)] + + def javascript_command(self, entrypoint: Path) -> List[str]: + if JAVASCRIPT_RUNNER is not None: + if not JAVASCRIPT_RUNNER.is_file() or not os.access(JAVASCRIPT_RUNNER, os.X_OK): + raise RuntimeError(f"JavaScript runner is not executable: {JAVASCRIPT_RUNNER}") + return [str(JAVASCRIPT_RUNNER), str(entrypoint)] + return [*self.node_command(), str(entrypoint)] + + def run_generated_typescript(self, root: Path, entrypoint: Path) -> subprocess.CompletedProcess[str]: + tsconfig = root / "tsconfig.json" + tsconfig.write_text( + """ + { + "compilerOptions": { + "target": "ES2019", + "module": "commonjs", + "strict": true, + "skipLibCheck": true, + "baseUrl": ".", + "rootDir": ".", + "outDir": "dist", + "lib": ["ES2019", "DOM"] + }, + "include": ["**/*.ts"] + } + """, + encoding="utf-8", + ) + + shim_dir = root / "client_sql" / "src" + shim_dir.mkdir(parents=True) + (shim_dir / "ClientSQLDebug.ts").write_text( + """ + export interface ClientSQLDebugDatabase {} + export const registeredDatabases: ClientSQLDebugDatabase[] = []; + export function registerClientSQLDebugDatabase(database: ClientSQLDebugDatabase): void { + registeredDatabases.push(database); + } + export function unregisterClientSQLDebugDatabase(database: ClientSQLDebugDatabase): void { + const index = registeredDatabases.indexOf(database); + if (index !== -1) { + registeredDatabases.splice(index, 1); + } + } + export function notifyClientSQLDebugChanged(_databaseName?: string): void {} + """, + encoding="utf-8", + ) + + compile_result = subprocess.run( + [*self.typescript_compiler_command(), "--project", str(tsconfig)], + cwd=root, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if compile_result.returncode != 0: + return compile_result + + compiled_entrypoint = root / "dist" / entrypoint.relative_to(root).with_suffix(".js") + env = dict(os.environ) + env["NODE_PATH"] = str(root / "dist") + return subprocess.run( + self.javascript_command(compiled_entrypoint), + cwd=root, + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def test_generates_typescript_database_and_query_bindings(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "User.sq").write_text( + """ + CREATE TABLE user ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + age INTEGER, + nickname TEXT + ); + + selectAll: + SELECT * FROM user; + + selectById: + SELECT id, name FROM user WHERE id = :id; + + insertUser: + INSERT INTO user(id, name, age) VALUES (:id, :name, :age); + """, + encoding="utf-8", + ) + migration_dir = sql_dir / "migration" + migration_dir.mkdir() + (migration_dir / "2.sqm").write_text("ALTER TABLE user ADD COLUMN nickname TEXT;", encoding="utf-8") + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + database = (out_dir / "TestDb.ts").read_text(encoding="utf-8") + self.assertIn("export class TestDb", database) + self.assertIn("openDatabase", database) + self.assertIn('const ALL_TABLES: string[] = ["user"];', database) + self.assertIn("watchQuery(", database) + self.assertIn("export type ClientSQLNativeCallback", database) + self.assertIn("client_sql/src/ClientSQLDebug", database) + self.assertIn("private debugDatabase: ClientSQLDebugDatabase | undefined;", database) + self.assertIn("registerClientSQLDebugDatabase(this.debugDatabase);", database) + self.assertIn("unregisterClientSQLDebugDatabase(this.debugDatabase);", database) + self.assertIn( + "execute(sql: string, parameters: ClientSQLValue[] | undefined, callback: ClientSQLNativeCallback): void;", + database, + ) + self.assertIn("export interface ClientSQLNativeTransaction", database) + self.assertIn( + "transaction(body: ClientSQLNativeTransactionBody, callback: ClientSQLNativeCallback): void;", + database, + ) + self.assertNotIn("queryOnWriter", database) + self.assertIn("function nativePromise", database) + self.assertIn("await nativePromise(callback => this.connection.execute(sql, parameters, callback));", database) + self.assertIn( + "nativePromise(callback => nativeTransaction.query(sql, parameters, callback))", + database, + ) + self.assertIn("debugInfo: (): Promise> => this.debugInfo(),", database) + self.assertIn("changedTables?: string[]", database) + self.assertIn("export interface TestDbTransaction", database) + self.assertIn("userQueries: new UserQueries(transactionDatabase)", database) + self.assertIn("this.emitChangedTables(changedTables);", database) + self.assertIn("export interface User", (out_dir / "UserTypes.ts").read_text(encoding="utf-8")) + self.assertFalse((out_dir / "TestDb.d.ts").exists()) + queries = (out_dir / "UserQueries.ts").read_text(encoding="utf-8") + self.assertIn("selectById(id: number)", queries) + self.assertIn("watchSelectById(id: number, listener: ClientSQLQueryListener)", queries) + self.assertIn('SELECT id, name FROM user WHERE id = ?;', queries) + self.assertIn("insertUser(id: number, name: string, age: number | null)", queries) + self.assertIn('["user"]', queries) + + def test_supports_sqldelight_style_query_shapes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "User.sq").write_text( + """ + CREATE TABLE user ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + age INTEGER + ); + + selectPage: + SELECT * FROM user ORDER BY name DESC LIMIT :limit OFFSET :offset; + + selectCommaPage: + SELECT * FROM user ORDER BY id LIMIT :rowOffset, :limit; + + selectByOptionalAge: + SELECT id, name FROM user WHERE (:age IS NULL OR age >= :age) ORDER BY id LIMIT :limit; + + countUsers: + SELECT count(*) AS count FROM user; + + updateUser: + UPDATE user SET name = :name, age = :age WHERE id = :id; + + deleteUser: + DELETE FROM user WHERE id = :id; + """, + encoding="utf-8", + ) + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + user_types = (out_dir / "UserTypes.ts").read_text(encoding="utf-8") + self.assertIn("export interface CountUsersRow", user_types) + self.assertIn("count: number;", user_types) + self.assertIn("export interface SelectPageParams", user_types) + self.assertIn("offset: number;", user_types) + self.assertIn("export interface SelectCommaPageParams", user_types) + self.assertIn("rowOffset: number;", user_types) + self.assertIn("export interface SelectByOptionalAgeParams", user_types) + + queries = (out_dir / "UserQueries.ts").read_text(encoding="utf-8") + self.assertIn("import { CountUsersRow, SelectByOptionalAgeRow, User } from './UserTypes';", queries) + self.assertNotIn("SelectByOptionalAgeParams", queries) + self.assertIn("selectPage(limit: number, offset: number): Promise", queries) + self.assertIn("watchSelectPage(limit: number, offset: number, listener: ClientSQLQueryListener)", queries) + self.assertIn('return this.db.watchQuery(["user"], () => this.selectPage(limit, offset), listener);', queries) + self.assertIn('SELECT * FROM user ORDER BY name DESC LIMIT ? OFFSET ?;', queries) + self.assertIn("selectCommaPage(rowOffset: number, limit: number): Promise", queries) + self.assertIn('SELECT * FROM user ORDER BY id LIMIT ?, ?;', queries) + self.assertIn("selectByOptionalAge(age: number | null, limit: number)", queries) + self.assertIn("updateUser(name: string, age: number | null, id: number): Promise", queries) + self.assertIn('], ["user"]);', queries) + self.assertIn("deleteUser(id: number): Promise", queries) + + def test_generated_reactive_contract_for_watchers_and_transactions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "User.sq").write_text( + """ + CREATE TABLE user ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL + ); + + selectAll: + SELECT * FROM user; + + insertUser: + INSERT INTO user(id, name) VALUES (:id, :name); + + updateUser: + UPDATE user SET name = :name WHERE id = :id; + + deleteUser: + DELETE FROM user WHERE id = :id; + """, + encoding="utf-8", + ) + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + database = (out_dir / "TestDb.ts").read_text(encoding="utf-8") + queries = (out_dir / "UserQueries.ts").read_text(encoding="utf-8") + + self.assertIn( + "const watchEntriesByDatabaseName: { [name: string]: ClientSQLWatchEntry[] | undefined } = " + "Object.create(null);", + database, + ) + self.assertIn(" unsubscribe(): void;", database) + self.assertIn( + "const writeChainsByDatabaseName: { [name: string]: Promise | undefined } = " + "Object.create(null);", + database, + ) + self.assertIn("function enqueueDatabaseWrite(name: string, body: () => Promise): Promise", database) + self.assertIn("const chain = previous.then(() => current, () => current);", database) + self.assertIn("writeChainsByDatabaseName[name] = chain;", database) + self.assertIn("releaseDatabaseWrite(name, chain, releaseCurrent);", database) + self.assertIn("return enqueueDatabaseWrite(this.databaseName, async () => {", database) + self.assertIn("notifyClientSQLDebugChanged", database) + self.assertIn("interface ClientSQLTransactionDebugEntry {", database) + self.assertIn("const MAX_TRANSACTION_DEBUG_ENTRIES = 50;", database) + self.assertIn("private nextTransactionDebugId = 1;", database) + self.assertIn("private transactionHistory: ClientSQLTransactionDebugEntry[] = [];", database) + self.assert_in_order( + database, + "entriesForDatabase(this.databaseName).push(entry);", + "this.localWatchEntries.push(entry);", + "emit();", + ) + self.assertIn("if (active && currentGeneration === generation) {", database) + self.assertIn("if (hasTableIntersection(entry.tables, changedTables)) {", database) + self.assertIn("const entry: ClientSQLWatchEntry = { tables, emit, unsubscribe };", database) + + self.assertIn( + 'return this.db.execute("INSERT INTO user(id, name) VALUES (?, ?);", [id, name], ["user"]);', + queries, + ) + self.assertIn( + 'return this.db.execute("UPDATE user SET name = ? WHERE id = ?;", [name, id], ["user"]);', + queries, + ) + self.assertIn('return this.db.execute("DELETE FROM user WHERE id = ?;", [id], ["user"]);', queries) + + transaction = database[database.index(" async transaction") : database.index(" async close")] + self.assert_in_order( + transaction, + "return enqueueDatabaseWrite(this.databaseName, () => this.runTransaction(body));", + "private async runTransaction", + "const changedTables: string[] = [];", + ) + self.assert_in_order( + transaction, + "const transactionDebugId = this.nextTransactionDebugId++;", + "const transactionStartedAtMs = Date.now();", + "this.connection.transaction((nativeTransaction, transactionCallback) => {", + "const transaction = this.createTransactionScope(nativeTransaction, changedTables);", + "void bodyPromise.then(", + "transactionCallback(undefined, undefined);", + "const transactionCompletedAtMs = Date.now();", + "this.recordTransactionDebugEntry({", + "status: 'committed',", + "this.emitChangedTables(changedTables);", + ) + self.assertIn("userQueries: new UserQueries(transactionDatabase)", transaction) + self.assertIn("transaction: nestedBody => nestedBody(scope)", transaction) + self.assertIn("ClientSQL watchers cannot be created inside a transaction", transaction) + self.assert_in_order( + transaction, + "} catch (error) {", + "this.recordTransactionDebugEntry({", + "status: 'rolled_back',", + "error: errorMessage(error),", + "notifyClientSQLDebugChanged(this.databaseName);", + "throw error;", + ) + self.assertNotIn("BEGIN TRANSACTION", transaction) + self.assertNotIn("COMMIT", transaction) + self.assertNotIn("ROLLBACK", transaction) + + close = database[database.index(" async close") : database.index(" private notifyTablesChanged")] + self.assertIn("if (typeof entry.unsubscribe === 'function') {", close) + self.assertIn("entry.unsubscribe();", close) + self.assertIn("removeWatchEntry(this.localWatchEntries, entry);", close) + self.assertIn( + "await enqueueDatabaseWrite(this.databaseName, () => nativePromise(callback => this.connection.close(callback)));", + close, + ) + + debug = database[database.index(" private async debugInfo") : database.index(" private notifyTablesChanged")] + self.assertIn("debugInfo?: (callback: ClientSQLNativeCallback>) => void;", debug) + self.assertIn("pendingChangedTableCount: this.activeTransactionChangedTables?.length ?? 0,", debug) + self.assertIn("transactionHistoryCount: this.transactionHistory.length,", debug) + self.assertIn("transactions: this.transactionHistory.slice().reverse(),", debug) + self.assertIn("watcherCount: (watchEntriesByDatabaseName[this.databaseName] || []).length,", debug) + + recorder = database[database.index(" private recordTransactionDebugEntry") : database.index(" private notifyTablesChanged")] + self.assertIn("this.transactionHistory.push(entry);", recorder) + self.assertIn("this.transactionHistory.length - MAX_TRANSACTION_DEBUG_ENTRIES", recorder) + + notify = database[database.index(" private notifyTablesChanged") : database.index(" private emitChangedTables")] + self.assertIn("this.emitChangedTables(uniqueTables);", notify) + self.assertNotIn("transactionDepth", notify) + + emit = database[database.index(" private emitChangedTables") : database.index("}", database.index(" private emitChangedTables"))] + self.assertIn("notifyClientSQLDebugChanged(this.databaseName);", emit) + + def test_generated_watchers_execute_reactive_contract(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "User.sq").write_text( + """ + CREATE TABLE user ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL + ); + + selectAll: + SELECT * FROM user ORDER BY id; + + countUsers: + SELECT count(*) AS count FROM user; + + insertUser: + INSERT INTO user(id, name) VALUES (:id, :name); + """, + encoding="utf-8", + ) + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + self.assertEqual(result.returncode, 0, msg=result.stderr) + + entrypoint = root / "generated_behavior_test.ts" + entrypoint.write_text( + r""" + import { + ClientSQLMigration, + ClientSQLNativeCallback, + ClientSQLNativeConnection, + ClientSQLNativeModule, + ClientSQLNativeTransaction, + ClientSQLNativeTransactionBody, + ClientSQLValue, + TestDb, + setClientSQLNativeForTests, + } from './out/TestDb'; + import { registeredDatabases } from 'client_sql/src/ClientSQLDebug'; + + interface UserRow { + id: number; + name: string; + } + + class Store { + rows: UserRow[] = []; + readonly operations: string[] = []; + } + + function queryRows(store: Store, sql: string): Record[] { + if (sql.indexOf('SELECT count(*) AS count FROM user') === 0) { + return [{ count: store.rows.length }]; + } + if (sql.indexOf('SELECT * FROM user') === 0) { + return store.rows + .slice() + .sort((left, right) => left.id - right.id) + .map(row => ({ ...row })); + } + throw new Error(`Unexpected query SQL: ${sql}`); + } + + class FakeTransaction implements ClientSQLNativeTransaction { + constructor(private readonly store: Store) {} + + execute( + sql: string, + parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + try { + this.store.operations.push(sql); + if (sql.indexOf('INSERT INTO user') === 0) { + const id = Number(parameters?.[0]); + const name = String(parameters?.[1]); + this.store.operations.push(`insert:${id}`); + this.store.rows.push({ id, name }); + } else { + throw new Error(`Unexpected execute SQL: ${sql}`); + } + callback(undefined, undefined); + } catch (error) { + callback(undefined, errorMessage(error)); + } + } + + query( + sql: string, + _parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + try { + callback(queryRows(this.store, sql) as T[], undefined); + } catch (error) { + callback(undefined, errorMessage(error)); + } + } + } + + class FakeConnection implements ClientSQLNativeConnection { + constructor(private readonly store: Store) {} + + execute( + sql: string, + parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + try { + this.store.operations.push(sql); + if (sql.indexOf('INSERT INTO user') === 0) { + const id = Number(parameters?.[0]); + const name = String(parameters?.[1]); + this.store.operations.push(`insert:${id}`); + this.store.rows.push({ id, name }); + } else { + throw new Error(`Unexpected execute SQL: ${sql}`); + } + callback(undefined, undefined); + } catch (error) { + callback(undefined, errorMessage(error)); + } + } + + query( + sql: string, + _parameters: ClientSQLValue[] | undefined, + callback: ClientSQLNativeCallback, + ): void { + try { + callback(queryRows(this.store, sql) as T[], undefined); + } catch (error) { + callback(undefined, errorMessage(error)); + } + } + + transaction( + body: ClientSQLNativeTransactionBody, + callback: ClientSQLNativeCallback, + ): void { + const snapshot = this.store.rows.map(row => ({ ...row })); + this.store.operations.push('transaction:start'); + body(new FakeTransaction(this.store), (_value, error) => { + if (error !== undefined && error !== null) { + this.store.rows = snapshot.map(row => ({ ...row })); + this.store.operations.push('transaction:rollback'); + callback(undefined, error); + return; + } + this.store.operations.push('transaction:commit'); + callback(undefined, undefined); + }); + } + + close(callback: ClientSQLNativeCallback): void { + this.store.operations.push('close'); + callback(undefined, undefined); + } + } + + class FakeNative implements ClientSQLNativeModule { + private readonly storesByName: { [name: string]: Store | undefined } = Object.create(null); + + openDatabase( + name: string, + _schemaVersion: number, + _createStatements: string[], + _migrations: ClientSQLMigration[], + ): ClientSQLNativeConnection { + return new FakeConnection(this.store(name)); + } + + store(name: string): Store { + let store = this.storesByName[name]; + if (!store) { + store = new Store(); + this.storesByName[name] = store; + } + return store; + } + } + + function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); + } + + function assert(condition: unknown, message: string): void { + if (!condition) { + throw new Error(message); + } + } + + function assertNames(rows: UserRow[], expected: string[], message: string): void { + const actual = rows.map(row => row.name).join(','); + const wanted = expected.join(','); + assert(actual === wanted, `${message}: expected ${wanted}, got ${actual}`); + } + + function nextTurn(): Promise { + return new Promise(resolve => setTimeout(resolve, 0)); + } + + async function waitFor(condition: () => boolean, message: string): Promise { + for (let attempt = 0; attempt < 20; attempt += 1) { + if (condition()) { + return; + } + await nextTurn(); + } + throw new Error(message); + } + + async function testWatcherEmissions(): Promise { + const native = new FakeNative(); + setClientSQLNativeForTests(native); + const db = TestDb.open('watcher-contract'); + const emissions: UserRow[][] = []; + const subscription = db.userQueries.watchSelectAll(rows => { + emissions.push(rows); + }); + + await waitFor(() => emissions.length === 1, 'initial watcher emission did not arrive'); + assertNames(emissions[0], [], 'initial watcher emission'); + + await db.userQueries.insertUser(1, 'Ada'); + await waitFor(() => emissions.length === 2, 'single write did not emit exactly once'); + assertNames(emissions[1], ['Ada'], 'single write watcher emission'); + + await db.transaction(async transaction => { + await transaction.userQueries.insertUser(2, 'Grace'); + await transaction.userQueries.insertUser(3, 'Katherine'); + }); + await waitFor(() => emissions.length === 3, 'transaction did not emit after commit'); + assertNames(emissions[2], ['Ada', 'Grace', 'Katherine'], 'transaction watcher emission'); + + let failed = false; + try { + await db.transaction(async transaction => { + await transaction.userQueries.insertUser(4, 'Rolled Back'); + throw new Error('rollback'); + }); + } catch { + failed = true; + } + assert(failed, 'rollback transaction should reject'); + await nextTurn(); + assert(emissions.length === 3, `rollback emitted ${emissions.length - 3} extra time(s)`); + + const countRows = await db.userQueries.countUsers(); + assert(countRows[0].count === 3, `rollback left ${countRows[0].count} rows`); + const debugInfo = await (registeredDatabases[0] as any).debugInfo(); + const transactions = debugInfo.transactions as Array<{ + status: string; + durationMs: number; + changedTableCount: number; + changedTables: string[]; + error?: string; + }>; + assert(transactions.length === 2, `expected 2 transaction history entries, got ${transactions.length}`); + assert(transactions[0].status === 'rolled_back', `newest transaction status was ${transactions[0].status}`); + assert(transactions[0].durationMs >= 0, 'rollback duration was not recorded'); + assert(transactions[0].changedTableCount === 1, 'rollback changed table count was not recorded'); + assert(transactions[0].changedTables[0] === 'user', 'rollback changed table was not recorded'); + assert(transactions[0].error === 'rollback', `rollback error was ${transactions[0].error}`); + assert(transactions[1].status === 'committed', `older transaction status was ${transactions[1].status}`); + assert(transactions[1].durationMs >= 0, 'commit duration was not recorded'); + assert(transactions[1].changedTableCount === 1, 'commit changed table count was not recorded'); + assert(transactions[1].changedTables[0] === 'user', 'commit changed table was not recorded'); + assert(debugInfo.queuedWrite === false, 'completed writes left the database queue marked active'); + + subscription.unsubscribe(); + await db.userQueries.insertUser(5, 'No Emit'); + await nextTurn(); + assert(emissions.length === 3, 'unsubscribed watcher emitted'); + await db.close(); + } + + async function testWriteQueueIsolation(): Promise { + const native = new FakeNative(); + setClientSQLNativeForTests(native); + const first = TestDb.open('queue-contract'); + const store = native.store('queue-contract'); + let releaseTransaction!: () => void; + let markStarted!: () => void; + const transactionStarted = new Promise(resolve => { + markStarted = resolve; + }); + const transactionBlocker = new Promise(resolve => { + releaseTransaction = resolve; + }); + + const transactionPromise = first.transaction(async transaction => { + await transaction.userQueries.insertUser(10, 'Inside'); + markStarted(); + await transactionBlocker; + }); + await transactionStarted; + + const activeDebugInfo = await (registeredDatabases[0] as any).debugInfo(); + assert(activeDebugInfo.queuedWrite === true, 'active transaction was not reported as queued'); + const outsideWrite = first.userQueries.insertUser(11, 'Outside'); + await nextTurn(); + await nextTurn(); + assert(!store.rows.some(row => row.id === 11), 'outside write interleaved into open transaction'); + + releaseTransaction(); + await transactionPromise; + await outsideWrite; + + const commitIndex = store.operations.indexOf('transaction:commit'); + const outsideIndex = store.operations.indexOf('insert:11'); + assert(commitIndex !== -1, 'transaction did not commit'); + assert(outsideIndex > commitIndex, `outside write ran before commit: ${store.operations.join(' | ')}`); + const idleDebugInfo = await (registeredDatabases[0] as any).debugInfo(); + assert(idleDebugInfo.queuedWrite === false, 'drained write queue remained marked active'); + await first.close(); + } + + async function main(): Promise { + await testWatcherEmissions(); + await testWriteQueueIsolation(); + } + + void main().catch(error => { + console.error(error); + throw error; + }); + """, + encoding="utf-8", + ) + + run_result = self.run_generated_typescript(root, entrypoint) + self.assertEqual( + run_result.returncode, + 0, + msg=f"stdout:\n{run_result.stdout}\nstderr:\n{run_result.stderr}", + ) + + def test_nested_sql_files_preserve_output_paths(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + nested_dir = sql_dir / "TestDb" / "account" + nested_dir.mkdir(parents=True) + (nested_dir / "Session.sq").write_text( + """ + CREATE TABLE session ( + id TEXT NOT NULL PRIMARY KEY, + created_at INTEGER NOT NULL + ); + + selectRecent: + SELECT * FROM session ORDER BY created_at DESC LIMIT :limit; + """, + encoding="utf-8", + ) + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + self.assertTrue((out_dir / "account" / "SessionTypes.ts").exists()) + self.assertTrue((out_dir / "account" / "SessionQueries.ts").exists()) + database = (out_dir / "TestDb.ts").read_text(encoding="utf-8") + self.assertIn("import { SessionQueries } from './account/SessionQueries';", database) + queries = (out_dir / "account" / "SessionQueries.ts").read_text(encoding="utf-8") + self.assertIn("import { Session } from './SessionTypes';", queries) + self.assertIn("selectRecent(limit: number): Promise", queries) + self.assertIn("watchSelectRecent(limit: number, listener: ClientSQLQueryListener)", queries) + + def test_validates_sql_and_tracks_all_reactive_tables(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "Feed.sq").write_text( + """ + CREATE TABLE author ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL + ); + + CREATE TABLE post ( + id INTEGER NOT NULL PRIMARY KEY, + author_id INTEGER NOT NULL REFERENCES author(id), + title TEXT NOT NULL + ); + + CREATE INDEX post_author_idx ON post(author_id); + + selectFeed: + SELECT post.id, post.title, author.name AS author_name + FROM post + JOIN author ON author.id = post.author_id + ORDER BY post.id; + """, + encoding="utf-8", + ) + + out_dir = root / "out" + result = self.run_clientsql(sql_dir, out_dir) + self.assertEqual(result.returncode, 0, msg=result.stderr) + + queries = (out_dir / "FeedQueries.ts").read_text(encoding="utf-8") + self.assertIn( + 'return this.db.watchQuery(["post", "author"], () => this.selectFeed(), listener);', + queries, + ) + database = (out_dir / "TestDb.ts").read_text(encoding="utf-8") + self.assertIn("CREATE INDEX post_author_idx ON post(author_id);", database) + + (db_dir / "Feed.sq").write_text( + """ + CREATE TABLE author (id INTEGER NOT NULL PRIMARY KEY); + + invalidQuery: + SELECT missing_column FROM author; + """, + encoding="utf-8", + ) + invalid_result = self.run_clientsql(sql_dir, root / "invalid-out") + self.assertNotEqual(invalid_result.returncode, 0) + self.assertIn("Invalid query Feed.sq:invalidQuery", invalid_result.stderr) + self.assertIn("missing_column", invalid_result.stderr) + + def test_rejects_duplicate_query_and_migration_versions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sql_dir = root / "sql" + db_dir = sql_dir / "TestDb" + db_dir.mkdir(parents=True) + (db_dir / "User.sq").write_text( + """ + CREATE TABLE user (id INTEGER NOT NULL PRIMARY KEY); + + selectAll: + SELECT * FROM user; + + selectAll: + SELECT id FROM user; + """, + encoding="utf-8", + ) + + duplicate_query = self.run_clientsql(sql_dir, root / "query-out") + self.assertNotEqual(duplicate_query.returncode, 0) + self.assertIn("Duplicate query name 'selectAll'", duplicate_query.stderr) + + (db_dir / "User.sq").write_text( + "CREATE TABLE user (id INTEGER NOT NULL PRIMARY KEY);", + encoding="utf-8", + ) + migration_dir = sql_dir / "migration" + migration_dir.mkdir() + (migration_dir / "2.sqm").write_text("ALTER TABLE user ADD COLUMN name TEXT;", encoding="utf-8") + (migration_dir / "2-extra.sqm").write_text("ALTER TABLE user ADD COLUMN age INTEGER;", encoding="utf-8") + + duplicate_migration = self.run_clientsql(sql_dir, root / "migration-out") + self.assertNotEqual(duplicate_migration.returncode, 0) + self.assertIn("Duplicate migration version 2", duplicate_migration.stderr) + + (migration_dir / "2.sqm").unlink() + (migration_dir / "2-extra.sqm").unlink() + (migration_dir / "3.sqm").write_text("ALTER TABLE user ADD COLUMN age INTEGER;", encoding="utf-8") + migration_gap = self.run_clientsql(sql_dir, root / "migration-gap-out") + self.assertNotEqual(migration_gap.returncode, 0) + self.assertIn("Migration versions must be contiguous starting at 2", migration_gap.stderr) + + def test_version_contract(self) -> None: + result = subprocess.run( + [*self.clientsql_command(), "-version"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0) + self.assertIn("valdi-clientsql", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift b/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift index 8b735079..11b223f6 100644 --- a/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift +++ b/compiler/compiler/Compiler/Sources/Processors/ClientSqlProcessor.swift @@ -95,11 +95,15 @@ final class ClientSqlProcessor: CompilationProcessor { ["-tm", Files.sqlTypesYaml] : [] // Locate the input files and expected output file path - let outputDirectory = projectConfig.generatedTsDirectoryURL + let unresolvedOutputDirectory = projectConfig.generatedTsDirectoryURL .appendingPathComponent("\(bundleInfo.name)") .appendingPathComponent(outdir) - .resolvingSymlinksInPath() - try fileManager.createDirectory(at: outputDirectory) + try fileManager.createDirectory(at: unresolvedOutputDirectory) + + // The generated directory may sit below Bazel's output symlink. Resolve + // it only after creation so emitted files and their base directory use + // the same canonical path on the first compilation pass. + let outputDirectory = unresolvedOutputDirectory.resolvingSymlinksInPath() // All input files sorted by path and put together as the key to locate the DiskCache entry let sortedSourcePaths = sourceMap.keys.sorted()