From 2e3d40bb1055b39e92b449afb1867f5ddd3458d6 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Fri, 11 Sep 2026 08:40:17 +0300 Subject: [PATCH 1/3] feat: MySQL and SQL Server adapters, and two engines that were never really checked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each is what the plan said it would be — a driver, a connection string, a catalog query and a capability list over the existing core — and the core grew two hooks to fit them. MySQL, which serves MariaDB. A database IS the schema, so discovery with no schema means the one this connection opened rather than every database on a shared server. No sequences, no MERGE, no RETURNING, no array type; a procedure returns rows by SELECTing, which is the capability PostgreSQL declares false. SQL Server, which serves Azure SQL. The default schema belongs to the LOGIN rather than the connection, so a data source that names one applies it per connection — through the new OnConnectionOpenedAsync hook, and not fatally, because impersonating a schema's owner is a privilege many service accounts do not have. MERGE, OUTPUT and snapshot isolation are all real here. The second hook is the one that mattered. The shared statement check prepares the SQL, and two engines cannot be asked that way: - SqlCommand.Prepare refuses unless every parameter has an explicit type, which a caller checking somebody else's SQL does not know. It asks sp_describe_undeclared_parameters instead. - ODP.NET's Prepare is a client-side NO-OP — Oracle compiles a statement when it is executed, not when it is prepared. So Oracle has been reporting every statement as valid without looking at one, including a select from a table that does not exist, and its connection test's "prepares every statement" was vacuous. It uses DBMS_SQL.PARSE now, which compiles and resolves names while running nothing, and the PL/SQL frames that raises are stripped so the answer is the one ORA- line about the operator's SQL. Two more things the engines made visible: - The schema browser wrote @name and `limit 100` whatever the connection was. That is right for two engines and wrong for the other two — Oracle binds :name and takes `fetch first`, SQL Server puts `top` before the column list. The adapter reports its own prefix and limit style now, so a draft cannot use a syntax the connection will refuse. It also drafted "call proc(@a)" for a procedure, which made a statement whose procedure NAME was that whole string. - A statement saved while its adapter was not running was never checked, and the save said nothing — indistinguishable from one that had been verified. Create and Update now answer with whether the database actually looked, and both forms say "Saved, but not checked" when it did not. 472 unit, 431 integration (35 new, across both engines) and 313 client tests. Co-Authored-By: Claude Opus 5 --- SW.Bitween.Adapters.Db.Core/DbContracts.cs | 20 + .../DbResidentAdapterBase.cs | 55 +- .../MySqlDbAdapter.cs | 494 ++++++++++++++ SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs | 74 +++ SW.Bitween.Adapters.Db.MySql/Program.cs | 13 + .../SW.Bitween.Adapters.Db.MySql.csproj | 24 + .../OracleDbAdapter.cs | 78 ++- SW.Bitween.Adapters.Db.SqlServer/Program.cs | 13 + .../SW.Bitween.Adapters.Db.SqlServer.csproj | 28 + .../SqlServerDbAdapter.cs | 619 ++++++++++++++++++ .../SqlServerOptions.cs | 80 +++ .../Resources/DataSourceStatements/Create.cs | 12 +- .../Resources/DataSourceStatements/Update.cs | 7 +- .../Fixtures/BitweenFixture.cs | 8 + .../Fixtures/BusAdapters.cs | 2 + .../Fixtures/MySqlDbFixture.cs | 125 ++++ .../Fixtures/SqlServerDbFixture.cs | 184 ++++++ .../SW.Bitween.IntegrationTests.csproj | 8 + .../Tests/DataSourceStatementTests.cs | 7 +- .../Tests/MySqlAdapterTests.cs | 540 +++++++++++++++ .../Tests/OracleAdapterTests.cs | 63 ++ .../Tests/SqlServerAdapterTests.cs | 565 ++++++++++++++++ SW.Bitween.Web/ClientApp/src/api/client.ts | 5 +- .../src/api/http/dataSourceStatements.ts | 21 +- .../src/pages/data-sources/SchemaBrowser.tsx | 12 +- .../src/pages/data-sources/Statements.tsx | 34 +- .../data-sources/__tests__/schema.test.ts | 53 +- .../src/pages/data-sources/schema.ts | 54 +- .../studio/DataSourceBinding.tsx | 14 +- SW.Bitween.sln | 28 + docs/database-adapters-api.md | 50 ++ tools/dev-database.md | 39 ++ tools/dev-warehouse-mysql.sql | 116 ++++ tools/dev-warehouse-oracle.sql | 119 ++++ tools/dev-warehouse-sqlserver.sql | 152 +++++ 35 files changed, 3664 insertions(+), 52 deletions(-) create mode 100644 SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs create mode 100644 SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs create mode 100644 SW.Bitween.Adapters.Db.MySql/Program.cs create mode 100644 SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj create mode 100644 SW.Bitween.Adapters.Db.SqlServer/Program.cs create mode 100644 SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj create mode 100644 SW.Bitween.Adapters.Db.SqlServer/SqlServerDbAdapter.cs create mode 100644 SW.Bitween.Adapters.Db.SqlServer/SqlServerOptions.cs create mode 100644 SW.Bitween.IntegrationTests/Fixtures/MySqlDbFixture.cs create mode 100644 SW.Bitween.IntegrationTests/Fixtures/SqlServerDbFixture.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/MySqlAdapterTests.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/SqlServerAdapterTests.cs create mode 100644 tools/dev-warehouse-mysql.sql create mode 100644 tools/dev-warehouse-oracle.sql create mode 100644 tools/dev-warehouse-sqlserver.sql diff --git a/SW.Bitween.Adapters.Db.Core/DbContracts.cs b/SW.Bitween.Adapters.Db.Core/DbContracts.cs index 0a9db6ae..ff8f02ed 100644 --- a/SW.Bitween.Adapters.Db.Core/DbContracts.cs +++ b/SW.Bitween.Adapters.Db.Core/DbContracts.cs @@ -70,6 +70,26 @@ public class DbCapabilities /// bulk, incrementing, timestamp, timestamp+incrementing, marker. public string[] ReceiveModes { get; set; } = Array.Empty(); + /// + /// How this engine writes a bind placeholder — : or @ — filled in by the base + /// from the adapter's own setting rather than declared per engine, so the two cannot disagree. + /// + /// Here because a caller that WRITES SQL needs it: the schema browser drafts a statement from + /// a table or a procedure, and a draft using the wrong prefix is a statement the engine refuses. + /// + public string ParameterPrefix { get; set; } + + /// + /// How this engine limits a result to the first N rows. Three shapes, and they are not + /// interchangeable: limit (PostgreSQL, MySQL) and fetchFirst (Oracle, and SQL + /// Server 2012+) go after the query, while top (SQL Server's idiom) goes before the + /// column list. + /// + /// Same reason as the prefix: a drafted statement carries a row limit, and the wrong one does + /// not parse. + /// + public string LimitStyle { get; set; } = "limit"; + /// Probed with the real credentials — what the engine allows AND this login has. public List Privileges { get; set; } = new(); diff --git a/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs b/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs index 3320a600..bd090c36 100644 --- a/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs +++ b/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs @@ -183,9 +183,25 @@ protected async Task OpenAsync(CancellationToken cancellationToken var connection = Factory.CreateConnection(); connection.ConnectionString = connectionString; await connection.OpenAsync(cancellationToken); + await OnConnectionOpenedAsync(connection, cancellationToken); return connection; } + /// + /// Runs once on every connection this adapter opens, before anything uses it. + /// + /// For the settings an engine will not take in a connection string. PostgreSQL puts search_path + /// there and Oracle takes CURRENT_SCHEMA the same way; SQL Server has neither, because its + /// default schema belongs to the login rather than to the connection — so it is the one that + /// needs this. + /// + /// A pooled connection carries whatever this did into its next use, which is the point: it is + /// paid once per physical connection rather than once per message. Anything set here therefore + /// has to be true for every caller of this data source, not for one of them. + /// + protected virtual Task OnConnectionOpenedAsync(DbConnection connection, + CancellationToken cancellationToken) => Task.CompletedTask; + DbCommand CreateCommand(DbConnection connection, string sql, IDictionary parameters, int? timeoutSeconds) { @@ -342,6 +358,10 @@ public virtual async Task Describe() } } + // Filled in here rather than declared per engine, so the prefix a caller is told to write + // and the prefix this adapter actually binds with cannot drift apart. + described.ParameterPrefix = ParameterPrefix; + described.Details["statements"] = string.Join(", ", statements.Names.OrderBy(n => n)); described.Details["allowAdHocSql"] = Options.AllowAdHocSql.ToString(); return described; @@ -613,17 +633,7 @@ static bool IsBareRoutineName(string sql) try { - using var command = connection.CreateCommand(); - command.CommandText = sql; - command.CommandTimeout = 10; - - // The placeholders have to be declared before Prepare, because some drivers validate - // that every parameter in the text has been supplied — Npgsql refuses outright — and - // a check that fell over on every parameterised statement would be worse than none. - DeclarePlaceholders(command); - PrepareCommand(command); - await Task.Run(() => command.Prepare()); - + await CheckSyntaxAsync(connection, sql); return (true, null); } catch (Exception ex) @@ -632,6 +642,29 @@ static bool IsBareRoutineName(string sql) } } + /// + /// Asks the engine to accept this SQL without running it. Throws when it will not. + /// + /// PREPARE is the portable form and is what most drivers want: the server parses the text and + /// binds every name in it, which is the whole check. It is overridable because one engine + /// cannot be asked that way — see the SQL Server adapter, whose driver refuses to prepare a + /// command unless every parameter has been given an explicit type, which is precisely what a + /// caller checking someone else's SQL does not know. + /// + protected virtual async Task CheckSyntaxAsync(DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.CommandTimeout = 10; + + // The placeholders have to be declared before Prepare, because some drivers validate that + // every parameter in the text has been supplied — Npgsql refuses outright — and a check + // that fell over on every parameterised statement would be worse than none. + DeclarePlaceholders(command); + PrepareCommand(command); + await Task.Run(() => command.Prepare()); + } + /// /// The one mistake worth naming rather than leaving to a position offset. /// diff --git a/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs b/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs new file mode 100644 index 00000000..c53fa7d5 --- /dev/null +++ b/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs @@ -0,0 +1,494 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MySqlConnector; +using SW.Serverless.Sdk; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Db.MySql; + +/// +/// Bitween's MySQL data source provider, which serves MariaDB too. +/// +/// Everything generic — the command surface, paging, the statement allow-list, the polling receiver +/// — lives in . What is here is the part that is genuinely +/// MySQL: the connection string, information_schema, and what this user is actually granted. +/// +/// The one structural difference from the other providers is that MySQL has no schema separate +/// from the database. A "schema" filter in discovery therefore means a database, and the default +/// is the one this connection opened rather than every database on the server — a service account +/// on a shared instance can usually see the names of databases it has no business reading. +/// +// Three roles, one package. "datasource" is what makes it configurable as a connection; +// "receiver" and "handler" are what put it in the pickers a subscription actually chooses from, +// because the same resident instance both polls a table and runs a statement on delivery. +[AdapterKind("datasource")] +[AdapterKind("receiver")] +[AdapterKind("handler")] +public class MySqlDbAdapter(IOptions options, ILogger logger) + : DbResidentAdapterBase(options.Value, logger) +{ + readonly MySqlOptions _options = options.Value; + + protected override DbProviderFactory Factory => MySqlConnectorFactory.Instance; + + /// + /// MySQL binds parameters as @name. ? is the wire protocol's own form and is + /// positional; the driver accepts named placeholders and matches them by name, which is what + /// every statement in Bitween is written against. + /// + protected override string ParameterPrefix => "@"; + + // ------------------------------------------------------------------ connection + + protected override string BuildConnectionString() + { + if (string.IsNullOrWhiteSpace(_options.Database)) + throw new InvalidOperationException( + "Database is required. In MySQL a database is also the schema, so this is what an " + + "unqualified table name in a statement will resolve against."); + + var builder = new MySqlConnectionStringBuilder + { + Server = _options.Host, + Port = (uint)Math.Max(1, _options.Port), + Database = _options.Database, + UserID = _options.UserName, + Password = _options.Password ?? "", + ConnectionTimeout = (uint)Math.Max(1, _options.ConnectTimeoutSeconds), + DefaultCommandTimeout = (uint)Math.Max(0, _options.CommandTimeoutSeconds), + ApplicationName = _options.ApplicationName ?? "Bitween", + + // The whole reason this adapter is resident: the pool outlives the message, so a + // connect, a TLS handshake and an authentication round trip are paid once rather than + // per Xchange. + Pooling = true, + MinimumPoolSize = (uint)Math.Max(0, _options.MinPoolSize), + MaximumPoolSize = (uint)Math.Max(1, _options.MaxPoolSize), + ConnectionIdleTimeout = (uint)Math.Max(0, _options.ConnectionIdleLifetimeSeconds), + + // A pooled connection MySQL closed at its end is indistinguishable from a healthy one + // until it is used. This is what turns that into a fresh connection rather than into a + // failed message — and it matters more here than anywhere, because wait_timeout on a + // managed instance is routinely minutes. + ConnectionReset = true, + + AllowZeroDateTime = _options.AllowZeroDateTime, + + // Server-side prepare, unless it is turned off. Also what makes the statement check on + // save a real check: IgnorePrepare true would have Prepare() do nothing locally and + // report every statement as valid. + IgnorePrepare = !_options.ServerPrepare + }; + + if (Enum.TryParse(_options.SslMode, ignoreCase: true, out var sslMode)) + builder.SslMode = sslMode; + else if (!string.IsNullOrWhiteSpace(_options.SslMode)) + throw new InvalidOperationException( + $"'{_options.SslMode}' is not a MySQL SSL mode. Use one of: " + + string.Join(", ", Enum.GetNames(typeof(MySqlSslMode))) + "."); + + return builder.ToString(); + } + + // ------------------------------------------------------------------ capabilities + + protected override DbCapabilities DescribeEngine() => new() + { + Engine = "MySQL", + + // No sequences: MySQL has AUTO_INCREMENT, which belongs to a column rather than being an + // object of its own. MariaDB does have them, and listing the type here for a server that + // is not MariaDB would offer a picker that always comes back empty. + SupportedObjects = ["table", "view", "procedure", "function"], + + StoredProcedures = true, + ProcedureOutParameters = true, + + // The one place MySQL is more capable than PostgreSQL here: a procedure returns result sets + // simply by SELECTing, with no cursor to declare and nothing to bind — so Call returns rows + // directly, where PostgreSQL needs a set-returning function queried with SELECT and Oracle + // needs an explicit REF CURSOR. + ProcedureResultSets = true, + MultipleResultSets = true, + + NamedParameters = true, + Transactions = true, + + // All four, and READ UNCOMMITTED genuinely does something here — unlike PostgreSQL, where + // it is accepted and silently treated as READ COMMITTED. + IsolationLevels = ["ReadUncommitted", "ReadCommitted", "RepeatableRead", "Serializable"], + + // MySqlBulkCopy exists and is fast, but BulkLoad is not implemented for it yet — false + // rather than advertising a path that would quietly fall back to row-by-row inserts. + BulkCopy = false, + + // No MERGE statement. The upsert is INSERT ... ON DUPLICATE KEY UPDATE, which is a + // different thing with different semantics, so this is false rather than approximately true. + Merge = false, + + // No RETURNING on MySQL. MariaDB has it for INSERT and DELETE; declaring it here would + // have a statement written against that fail on the majority of servers. + Returning = false, + + Json = true, + + // No array type. JSON arrays are the usual stand-in, and they arrive as a string. + ArrayTypes = false, + + ChangeNotification = false, + LogBasedCdc = false, + + SchemaDiscovery = true, + RowCountEstimates = true, + ReceiveModes = ["bulk", "incrementing", "timestamp", "timestamp+incrementing", "marker"] + }; + + /// + /// What this user may do, asked of the server rather than assumed. SHOW GRANTS is the only + /// answer that accounts for roles, wildcards and grants made at every level at once — + /// information_schema.user_privileges reports the global ones only. + /// + protected override async Task> ProbePrivilegesAsync(DbConnection connection, + CancellationToken cancellationToken) + { + var privileges = new List(); + + try + { + using var command = connection.CreateCommand(); + command.CommandText = "show grants for current_user()"; + command.CommandTimeout = 10; + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var grant = reader.GetString(0); + + // "GRANT SELECT, INSERT ON `sales`.* TO ..." — the useful half is between GRANT + // and ON, and the rest is the grantee, which is who we already asked about. + var on = grant.IndexOf(" ON ", StringComparison.OrdinalIgnoreCase); + if (!grant.StartsWith("GRANT ", StringComparison.OrdinalIgnoreCase) || on < 0) + { + privileges.Add(grant); + continue; + } + + var what = grant.Substring(6, on - 6).Trim(); + var where = grant.Substring(on + 4).Trim(); + var to = where.IndexOf(" TO ", StringComparison.OrdinalIgnoreCase); + if (to > 0) where = where.Substring(0, to).Trim(); + + privileges.Add($"{what} on {where}"); + } + } + catch (Exception ex) + { + // A user that cannot run SHOW GRANTS is unusual but not broken — it just cannot tell + // us what it can do. Saying so beats failing the connection test over it. + Logger.LogDebug(ex, "Could not read grants."); + privileges.Add($"(could not be read: {ex.Message})"); + } + + return privileges; + } + + protected override IEnumerable> ExtraStatusDetails() + { + yield return new KeyValuePair("mysql.database", _options.Database ?? ""); + yield return new KeyValuePair("mysql.sslMode", _options.SslMode ?? ""); + yield return new KeyValuePair("mysql.serverPrepare", _options.ServerPrepare.ToString()); + } + + // ------------------------------------------------------------------ discovery + + /// + /// information_schema, which on MySQL is the catalog rather than a slow standard view over one + /// — there is no lower layer to reach for the way pg_class is under PostgreSQL's. + /// + protected override async Task> DiscoverAsync(DbConnection connection, + DiscoverRequest request, CancellationToken cancellationToken) + { + // A schema IS a database here, and the default is the one this connection opened. Listing + // every database on the server instead would be a longer answer and a worse one: a service + // account on a shared instance can often see names it has no business reading. + var schema = request.Schema ?? _options.Database; + var like = request.NameLike?.ToLowerInvariant(); + + return request.ObjectType switch + { + "procedure" or "function" => + await RoutinesAsync(connection, request, schema, like, cancellationToken), + _ => await RelationsAsync(connection, request, schema, like, cancellationToken) + }; + } + + async Task> RelationsAsync(DbConnection connection, DiscoverRequest request, + string schema, string like, CancellationToken cancellationToken) + { + var wanted = request.ObjectType == "view" ? "VIEW" : "BASE TABLE"; + + var parameters = new Dictionary { ["wanted"] = wanted }; + var sql = new StringBuilder(@" + select t.table_schema, t.table_name, t.table_type, t.table_comment, t.table_rows + from information_schema.tables t + where t.table_type = @wanted + and t.table_schema not in ('information_schema', 'mysql', 'performance_schema', 'sys')"); + + if (schema != null) { sql.Append(" and t.table_schema = @schema"); parameters["schema"] = schema; } + if (like != null) { sql.Append(" and locate(@nameLike, lower(t.table_name)) > 0"); parameters["nameLike"] = like; } + + sql.Append(" order by t.table_schema, t.table_name limit @take offset @skip"); + parameters["skip"] = request.Skip; + parameters["take"] = request.Take; + + var objects = new List(); + using (var command = connection.CreateCommand()) + { + command.CommandText = sql.ToString(); + command.CommandTimeout = Options.CommandTimeoutSeconds; + AddParameters(command, parameters); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + objects.Add(new DbObject + { + Schema = reader.GetString(0), + Name = reader.GetString(1), + Type = string.Equals(reader.GetString(2), "VIEW", StringComparison.OrdinalIgnoreCase) + ? "view" + : "table", + Comment = reader.IsDBNull(3) || reader.GetString(3).Length == 0 + ? null + : reader.GetString(3), + + // table_rows is InnoDB's estimate from the index statistics and can be out by + // a wide margin on a table that has not been analysed. Reported as an estimate + // everywhere it surfaces — the alternative is COUNT(*) on a stranger's table, + // which a menu should not do. Always null for a view, which has no rows of its + // own to estimate. + RowCount = !request.IncludeRowCounts || reader.IsDBNull(4) + ? null + : Math.Max(0, reader.GetInt64(4)) + }); + } + + if (request.IncludeColumns && objects.Count > 0) + await FillColumnsAsync(connection, objects, cancellationToken); + + return objects; + } + + async Task FillColumnsAsync(DbConnection connection, List objects, + CancellationToken cancellationToken) + { + // One query for the whole page, not one per object: a page of 200 tables would otherwise + // be 200 round trips, and against a remote database that is the difference between a + // screen that opens and one that times out. + // + // MySqlConnector has no array parameter, so the pair of IN lists is built from the page's + // own values. They are identifiers this connection just read back from the catalog, not + // anything a caller supplied — but they are quoted anyway, because "it cannot contain a + // quote" is the kind of assumption that survives right up until a table is named oddly. + var schemas = string.Join(",", objects.Select(o => Quote(o.Schema)).Distinct()); + var names = string.Join(",", objects.Select(o => Quote(o.Name)).Distinct()); + + using var command = connection.CreateCommand(); + command.CommandText = $@" + select c.table_schema, c.table_name, c.column_name, c.column_type, + c.is_nullable, c.ordinal_position, c.extra, c.column_key, + c.character_maximum_length, c.numeric_precision, c.numeric_scale + from information_schema.columns c + where c.table_schema in ({schemas}) and c.table_name in ({names}) + order by c.table_schema, c.table_name, c.ordinal_position"; + command.CommandTimeout = Options.CommandTimeoutSeconds; + + var byObject = objects.ToDictionary(o => $"{o.Schema}.{o.Name}"); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = $"{reader.GetString(0)}.{reader.GetString(1)}"; + if (!byObject.TryGetValue(key, out var target)) continue; + + var dbType = reader.GetString(3); + var extra = reader.IsDBNull(6) ? "" : reader.GetString(6); + + target.Columns.Add(new DbColumn + { + Name = reader.GetString(2), + DbType = dbType, + ClrType = ClrTypeOf(dbType), + Nullable = string.Equals(reader.GetString(4), "YES", StringComparison.OrdinalIgnoreCase), + Ordinal = (int)reader.GetInt64(5), + + // auto_increment, a generated column, and DEFAULT_GENERATED — which is what a + // CURRENT_TIMESTAMP default reports as. All three are values the database fills in, + // which is the only distinction an insert cares about. + Generated = extra.IndexOf("auto_increment", StringComparison.OrdinalIgnoreCase) >= 0 + || extra.IndexOf("GENERATED", StringComparison.OrdinalIgnoreCase) >= 0, + + // PRI on every column of the key, including each column of a composite one. + PrimaryKey = !reader.IsDBNull(7) + && string.Equals(reader.GetString(7), "PRI", StringComparison.OrdinalIgnoreCase), + + Length = reader.IsDBNull(8) ? null : (int?)reader.GetInt64(8), + Precision = reader.IsDBNull(9) ? null : (int?)reader.GetInt64(9), + Scale = reader.IsDBNull(10) ? null : (int?)reader.GetInt64(10) + }); + } + } + + async Task> RoutinesAsync(DbConnection connection, DiscoverRequest request, + string schema, string like, CancellationToken cancellationToken) + { + var wanted = request.ObjectType == "procedure" ? "PROCEDURE" : "FUNCTION"; + + var parameters = new Dictionary { ["wanted"] = wanted }; + var sql = new StringBuilder(@" + select r.routine_schema, r.routine_name, r.routine_type, r.routine_comment, r.dtd_identifier + from information_schema.routines r + where r.routine_type = @wanted + and r.routine_schema not in ('information_schema', 'mysql', 'performance_schema', 'sys')"); + + if (schema != null) { sql.Append(" and r.routine_schema = @schema"); parameters["schema"] = schema; } + if (like != null) { sql.Append(" and locate(@nameLike, lower(r.routine_name)) > 0"); parameters["nameLike"] = like; } + + sql.Append(" order by r.routine_schema, r.routine_name limit @take offset @skip"); + parameters["skip"] = request.Skip; + parameters["take"] = request.Take; + + var objects = new List(); + using (var command = connection.CreateCommand()) + { + command.CommandText = sql.ToString(); + command.CommandTimeout = Options.CommandTimeoutSeconds; + AddParameters(command, parameters); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var routine = new DbObject + { + Schema = reader.GetString(0), + Name = reader.GetString(1), + Type = string.Equals(reader.GetString(2), "PROCEDURE", StringComparison.OrdinalIgnoreCase) + ? "procedure" + : "function", + Comment = reader.IsDBNull(3) || reader.GetString(3).Length == 0 + ? null + : reader.GetString(3) + }; + + // dtd_identifier is the return type, and only a function has one. + if (routine.Type == "function" && !reader.IsDBNull(4)) + routine.Parameters.Add(new DbRoutineParameter + { + Name = "(returns)", + DbType = reader.GetString(4), + Direction = "ReturnValue", + Ordinal = 0 + }); + + objects.Add(routine); + } + } + + if (objects.Count > 0) await FillParametersAsync(connection, objects, cancellationToken); + return objects; + } + + async Task FillParametersAsync(DbConnection connection, List routines, + CancellationToken cancellationToken) + { + // information_schema.parameters, one query for the page. The row with ordinal_position 0 + // is a function's return value, which RoutinesAsync has already recorded from + // dtd_identifier — so it is skipped here rather than listed twice. + var schemas = string.Join(",", routines.Select(r => Quote(r.Schema)).Distinct()); + var names = string.Join(",", routines.Select(r => Quote(r.Name)).Distinct()); + + using var command = connection.CreateCommand(); + command.CommandText = $@" + select p.specific_schema, p.specific_name, p.parameter_name, + p.dtd_identifier, p.parameter_mode, p.ordinal_position + from information_schema.parameters p + where p.specific_schema in ({schemas}) and p.specific_name in ({names}) + and p.ordinal_position > 0 + order by p.specific_schema, p.specific_name, p.ordinal_position"; + command.CommandTimeout = Options.CommandTimeoutSeconds; + + var byRoutine = routines.ToDictionary(r => $"{r.Schema}.{r.Name}"); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = $"{reader.GetString(0)}.{reader.GetString(1)}"; + if (!byRoutine.TryGetValue(key, out var target)) continue; + + var mode = reader.IsDBNull(4) ? "IN" : reader.GetString(4); + target.Parameters.Add(new DbRoutineParameter + { + Name = reader.IsDBNull(2) ? $"p{reader.GetInt32(5)}" : reader.GetString(2), + DbType = reader.IsDBNull(3) ? "" : reader.GetString(3), + Direction = mode.ToUpperInvariant() switch + { + "OUT" => "Out", + "INOUT" => "InOut", + _ => "In" + }, + Ordinal = reader.GetInt32(5) + }); + } + } + + /// + /// A single-quoted literal for the IN lists above. Doubling the quote is MySQL's own escape and + /// is what the driver would do for a parameter. + /// + static string Quote(string value) => "'" + (value ?? "").Replace("'", "''").Replace("\\", "\\\\") + "'"; + + /// + /// What a value of this column arrives as in a result row. Coarse on purpose — it tells a + /// mapper author whether to expect a string or a number, and is not trying to be a type system. + /// + static string ClrTypeOf(string dbType) + { + var bare = dbType.Split('(')[0].Trim().ToLowerInvariant(); + var unsigned = dbType.IndexOf("unsigned", StringComparison.OrdinalIgnoreCase) >= 0; + + return bare switch + { + // tinyint(1) is how MySQL stores a boolean — there is no separate type — and the + // driver hands it back as one, so saying "int" here would mislead a mapper author. + "tinyint" when dbType.StartsWith("tinyint(1)", StringComparison.OrdinalIgnoreCase) => "bool", + "bool" or "boolean" => "bool", + "tinyint" or "smallint" or "mediumint" or "int" or "integer" => unsigned ? "long" : "int", + "bigint" => unsigned ? "decimal" : "long", + "decimal" or "numeric" => "decimal", + "float" or "double" or "real" => "double", + "date" or "datetime" or "timestamp" => "DateTime", + "time" => "TimeSpan", + "year" => "int", + "bit" => "bool", + "json" => "string", + "binary" or "varbinary" or "tinyblob" or "blob" or "mediumblob" or "longblob" => "byte[]", + _ => "string" + }; + } + + void AddParameters(DbCommand command, Dictionary parameters) + { + foreach (var kv in parameters) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = kv.Key; + parameter.Value = kv.Value ?? DBNull.Value; + command.Parameters.Add(parameter); + } + } +} diff --git a/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs b/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs new file mode 100644 index 00000000..3f8fdefc --- /dev/null +++ b/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs @@ -0,0 +1,74 @@ +using SW.Bitween.Adapters; + +namespace SW.Bitween.Adapters.Db.MySql; + +/// +/// Every setting here arrives as a DataSource property, bound by name, and the form an operator +/// fills in is generated from these attributes — so a field added here appears in Bitween with no +/// front-end change. +/// +/// The hints spend their weight on the two things that are genuinely MySQL's own: a database and +/// a schema are the same thing, which surprises anyone arriving from PostgreSQL or Oracle, and +/// SSL mode, which is what people get wrong against a managed instance. +/// +[AdapterSettings( + Kind = "Relational", + Label = "MySQL", + Description = "A MySQL or MariaDB database, held open with a pooled connection so statements, " + + "procedures and polling receivers do not pay a connect on every message.")] +public class MySqlOptions : DbOptionsBase +{ + [AdapterSetting(Required = true, Hint = "Host name or IP. No protocol prefix, no mysql:// URL.")] + public string Host { get; set; } = "localhost"; + + [AdapterSetting(Default = "3306")] + public int Port { get; set; } = 3306; + + /// + /// MySQL has no schema separate from the database — CREATE SCHEMA is a synonym for CREATE + /// DATABASE — so this one value is both, and it is what an unqualified table name resolves + /// against. + /// + [AdapterSetting(Required = true, Hint = + "The database to connect to. In MySQL a database IS a schema, so this is also what an " + + "unqualified table name resolves against — there is no separate schema setting.")] + public string Database { get; set; } + + [AdapterSetting(Required = true)] + public string UserName { get; set; } + + [AdapterSetting(Secret = true, Required = true)] + public string Password { get; set; } + + [AdapterSetting(Default = "Preferred", + AllowedValues = new[] { "None", "Preferred", "Required", "VerifyCA", "VerifyFull" }, + Hint = "Required or above for anything that is not localhost. Preferred will silently fall " + + "back to an unencrypted connection if the server does not offer TLS, which is " + + "exactly the case you wanted to know about.")] + public string SslMode { get; set; } = "Preferred"; + + [AdapterSetting(Default = "Bitween", Hint = + "What this connection calls itself in the performance schema. Worth keeping distinctive: " + + "it is how a DBA works out which of the connections on their server is yours.")] + public string ApplicationName { get; set; } = "Bitween"; + + [AdapterSetting(Default = "180", Hint = + "Seconds an idle pooled connection is kept before it is closed. Keep it below the server's " + + "own wait_timeout — the default is 28800, but a managed instance or a proxy in front of " + + "one is often far lower, and a connection the server closed first surfaces as a broken " + + "pipe on the next message rather than as a timeout.")] + public int ConnectionIdleLifetimeSeconds { get; set; } = 180; + + [AdapterSetting(Default = "true", AllowedValues = new[] { "true", "false" }, Hint = + "Ask the server to parse and plan a statement once, then run it by handle. Worth leaving " + + "on for a subscription, which runs the same statement over and over. Turn it off for a " + + "connection through a proxy such as ProxySQL, where prepared statements are pinned to a " + + "backend and can defeat the pooling the proxy is there to provide.")] + public bool ServerPrepare { get; set; } = true; + + [AdapterSetting(Default = "false", AllowedValues = new[] { "true", "false" }, Hint = + "Return DATE and DATETIME as strings rather than dates. MySQL permits '0000-00-00', which " + + "is not a date any .NET type can hold — a column holding one throws on read unless this " + + "is on. Only needed against a schema that has them, which is usually an old one.")] + public bool AllowZeroDateTime { get; set; } +} diff --git a/SW.Bitween.Adapters.Db.MySql/Program.cs b/SW.Bitween.Adapters.Db.MySql/Program.cs new file mode 100644 index 00000000..a1fa9937 --- /dev/null +++ b/SW.Bitween.Adapters.Db.MySql/Program.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.DependencyInjection; +using SW.Serverless.Sdk.Hosting; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Db.MySql; + +static class Program +{ + static Task Main() => AdapterHost.CreateBuilder() + .ConfigureServices((configuration, services) => services.Configure(configuration)) + .Build() + .RunResidentAsync(); +} diff --git a/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj b/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj new file mode 100644 index 00000000..10f66a09 --- /dev/null +++ b/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj @@ -0,0 +1,24 @@ + + + + Exe + net8.0 + SW.Bitween.Adapters.Db.MySql + disable + + + + + + + + + + + + + diff --git a/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs b/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs index 62015b7a..b50337a1 100644 --- a/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs +++ b/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs @@ -124,6 +124,78 @@ protected override void PrepareCommand(DbCommand command) // ------------------------------------------------------------------ capabilities + // ------------------------------------------------------------------ checking + + /// + /// Oracle is checked with DBMS_SQL.PARSE rather than by preparing. + /// + /// ODP.NET's Prepare is a client-side no-op — Oracle validates when a statement is + /// executed, not when it is prepared — so the shared check passed everything here, including a + /// select from a table that does not exist. A statement reported as verified without being + /// looked at is worse than no check, because it is the one nobody goes back to. + /// + /// PARSE is the real thing: it compiles the statement and resolves every name in it, raising + /// ORA-00942 for a missing table and ORA-00936 for a syntax error, and it runs nothing. Bind + /// placeholders need no values — parsing is exactly the step before binding. + /// + protected override async Task CheckSyntaxAsync(DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = @" + declare + c integer := dbms_sql.open_cursor; + begin + begin + dbms_sql.parse(c, :statement, dbms_sql.native); + exception + when others then + -- Closed here as well as below: leaving a cursor open on the shared pooled + -- connection would leak one per bad statement somebody tried to save. + dbms_sql.close_cursor(c); + raise; + end; + dbms_sql.close_cursor(c); + end;"; + command.CommandTimeout = 10; + + var parameter = command.CreateParameter(); + parameter.ParameterName = "statement"; + parameter.Value = sql; + command.Parameters.Add(parameter); + + PrepareCommand(command); + + try + { + await command.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + // Rethrown without the PL/SQL stack. Running the parse inside a block means the driver + // reports where in OUR block it failed — three ORA-06512 frames through SYS.DBMS_SQL — + // underneath the one line that is about the operator's SQL. Keeping them would bury + // "table does not exist" under the mechanism used to discover it. + throw new InvalidOperationException(FirstOracleError(ex.Message), ex); + } + } + + /// + /// The first ORA- line, which is the cause; the rest are the frames it was raised through. + /// Anything that does not look like an Oracle error is returned whole rather than trimmed to + /// nothing. + /// + static string FirstOracleError(string message) + { + if (string.IsNullOrWhiteSpace(message)) return message; + + var lines = message.Split('\n', StringSplitOptions.RemoveEmptyEntries); + var cause = lines.FirstOrDefault(l => + l.TrimStart().StartsWith("ORA-", StringComparison.OrdinalIgnoreCase) + && !l.TrimStart().StartsWith("ORA-06512", StringComparison.OrdinalIgnoreCase)); + + return (cause ?? lines[0]).Trim(); + } + protected override DbCapabilities DescribeEngine() => new() { Engine = "Oracle", @@ -157,7 +229,11 @@ protected override void PrepareCommand(DbCommand command) SchemaDiscovery = true, RowCountEstimates = true, - ReceiveModes = ["bulk", "incrementing", "timestamp", "timestamp+incrementing", "marker"] + ReceiveModes = ["bulk", "incrementing", "timestamp", "timestamp+incrementing", "marker"], + + // `fetch first N rows only`, since 12c. `rownum` is the older idiom and is a trap with an + // ORDER BY — it is applied before the sort, so it takes an arbitrary N and then sorts those. + LimitStyle = "fetchFirst" }; /// diff --git a/SW.Bitween.Adapters.Db.SqlServer/Program.cs b/SW.Bitween.Adapters.Db.SqlServer/Program.cs new file mode 100644 index 00000000..60eccd30 --- /dev/null +++ b/SW.Bitween.Adapters.Db.SqlServer/Program.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.DependencyInjection; +using SW.Serverless.Sdk.Hosting; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Db.SqlServer; + +static class Program +{ + static Task Main() => AdapterHost.CreateBuilder() + .ConfigureServices((configuration, services) => services.Configure(configuration)) + .Build() + .RunResidentAsync(); +} diff --git a/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj b/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj new file mode 100644 index 00000000..5a1650eb --- /dev/null +++ b/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj @@ -0,0 +1,28 @@ + + + + Exe + net8.0 + SW.Bitween.Adapters.Db.SqlServer + disable + + + + + + + + + + + + + diff --git a/SW.Bitween.Adapters.Db.SqlServer/SqlServerDbAdapter.cs b/SW.Bitween.Adapters.Db.SqlServer/SqlServerDbAdapter.cs new file mode 100644 index 00000000..a42071cc --- /dev/null +++ b/SW.Bitween.Adapters.Db.SqlServer/SqlServerDbAdapter.cs @@ -0,0 +1,619 @@ +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SW.Serverless.Sdk; +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Db.SqlServer; + +/// +/// Bitween's SQL Server data source provider, which serves Azure SQL too. +/// +/// Everything generic — the command surface, paging, the statement allow-list, the polling receiver +/// — lives in . What is here is the part that is genuinely SQL +/// Server: the connection string, the sys catalog views, and what this principal is granted. +/// +// Three roles, one package. "datasource" is what makes it configurable as a connection; +// "receiver" and "handler" are what put it in the pickers a subscription actually chooses from, +// because the same resident instance both polls a table and runs a statement on delivery. +[AdapterKind("datasource")] +[AdapterKind("receiver")] +[AdapterKind("handler")] +public class SqlServerDbAdapter(IOptions options, ILogger logger) + : DbResidentAdapterBase(options.Value, logger) +{ + readonly SqlServerOptions _options = options.Value; + + protected override DbProviderFactory Factory => SqlClientFactory.Instance; + + protected override string ParameterPrefix => "@"; + + // ------------------------------------------------------------------ connection + + protected override string BuildConnectionString() + { + if (string.IsNullOrWhiteSpace(_options.Database)) + throw new InvalidOperationException( + "Database is required: this connects to one database on the server, and it is what " + + "an unqualified name in a statement resolves against."); + + // A named instance is resolved through the SQL Server Browser, not by port, and supplying + // both is an error the driver reports obscurely — so the port is only added when the host + // is not already naming an instance. + var host = (_options.Host ?? "").Trim(); + var dataSource = host.Contains('\\') || host.Contains(',') + ? host + : $"{host},{Math.Max(1, _options.Port)}"; + + var builder = new SqlConnectionStringBuilder + { + DataSource = dataSource, + InitialCatalog = _options.Database, + UserID = _options.UserName, + Password = _options.Password ?? "", + ConnectTimeout = Math.Max(1, _options.ConnectTimeoutSeconds), + CommandTimeout = Math.Max(0, _options.CommandTimeoutSeconds), + ApplicationName = _options.ApplicationName ?? "Bitween", + + // The whole reason this adapter is resident: the pool outlives the message, so a + // connect, a TLS handshake and an authentication round trip are paid once rather than + // per Xchange. + Pooling = true, + MinPoolSize = Math.Max(0, _options.MinPoolSize), + MaxPoolSize = Math.Max(1, _options.MaxPoolSize), + + Encrypt = _options.Encrypt, + TrustServerCertificate = _options.TrustServerCertificate, + MultipleActiveResultSets = _options.MultipleActiveResultSets + }; + + return builder.ToString(); + } + + /// + /// SQL Server has no search_path: the default schema belongs to the LOGIN, not to the + /// connection. Where a data source names one, it is applied per connection instead. + /// + /// Failure is deliberately not fatal. Impersonating a schema's owner is a privilege many + /// service accounts do not have, and a connection that works perfectly for qualified names + /// should not be refused because it could not take a shortcut for unqualified ones. + /// + protected override async Task OnConnectionOpenedAsync(DbConnection connection, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(_options.Schema) && !_options.SnapshotIsolation) return; + + try + { + if (_options.SnapshotIsolation) + { + using var isolation = connection.CreateCommand(); + isolation.CommandText = "set transaction isolation level snapshot"; + isolation.CommandTimeout = 10; + await isolation.ExecuteNonQueryAsync(cancellationToken); + } + + if (!string.IsNullOrWhiteSpace(_options.Schema)) + { + using var command = connection.CreateCommand(); + + // The owner of the schema, which is the principal whose default schema it is. Bound + // as a parameter and then emitted through QUOTENAME, because EXECUTE AS takes a + // literal and this value comes from configuration. + command.CommandText = @" + declare @owner sysname = + (select dp.name + from sys.schemas s + join sys.database_principals dp on dp.principal_id = s.principal_id + where s.name = @schema); + if @owner is not null and @owner <> user_name() + exec('execute as user = ' + quotename(@owner));"; + command.CommandTimeout = 10; + + var parameter = command.CreateParameter(); + parameter.ParameterName = "schema"; + parameter.Value = _options.Schema; + command.Parameters.Add(parameter); + + await command.ExecuteNonQueryAsync(cancellationToken); + } + } + catch (Exception ex) + { + Logger.LogDebug(ex, + "Could not apply the connection defaults for schema {Schema}; unqualified names " + + "will resolve against the login's own default schema.", _options.Schema); + } + } + + // ------------------------------------------------------------------ checking + + /// + /// SQL Server is checked with sp_describe_undeclared_parameters rather than by preparing. + /// + /// SqlCommand.Prepare refuses unless every parameter has been given an explicit type and + /// size — which is exactly what a caller checking somebody else's SQL does not know, and + /// guessing nvarchar(4000) for a parameter compared against an int would make the check answer + /// a different question from the one asked. + /// + /// The procedure parses the batch and binds every name in it, then reports the parameters it + /// would need — so it catches a syntax error and a missing table, which are the two things this + /// check exists to catch, and needs nothing supplied. + /// + protected override async Task CheckSyntaxAsync(DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = "sys.sp_describe_undeclared_parameters"; + command.CommandType = CommandType.StoredProcedure; + command.CommandTimeout = 10; + + var parameter = command.CreateParameter(); + parameter.ParameterName = "@tsql"; + parameter.DbType = DbType.String; + parameter.Size = -1; + parameter.Value = sql; + command.Parameters.Add(parameter); + + // ExecuteReader rather than ExecuteNonQuery: the answer is a result set, and a batch that + // will not bind throws while producing it. + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) { } + } + + // ------------------------------------------------------------------ capabilities + + protected override DbCapabilities DescribeEngine() => new() + { + Engine = "SQL Server", + SupportedObjects = ["table", "view", "procedure", "function", "sequence"], + + StoredProcedures = true, + ProcedureOutParameters = true, + + // A procedure returns result sets by SELECTing, with nothing to declare and nothing to + // bind — so Call returns rows directly, where PostgreSQL needs a set-returning function + // queried with SELECT and Oracle needs an explicit REF CURSOR. + ProcedureResultSets = true, + MultipleResultSets = true, + + NamedParameters = true, + Transactions = true, + + // Snapshot as well as the four standard levels, and it is the one worth having here: a + // long-running read for a polling receiver does not block the writers it is reading from. + IsolationLevels = + ["ReadUncommitted", "ReadCommitted", "RepeatableRead", "Serializable", "Snapshot"], + + // SqlBulkCopy exists and is the fastest path there is, but BulkLoad is not implemented for + // it yet — false rather than advertising a path that would quietly fall back to row-by-row. + BulkCopy = false, + + // The real thing, not an approximation of one. + Merge = true, + + // The OUTPUT clause, which does what RETURNING does and does it for MERGE too. + Returning = true, + + Json = true, + + // Table-valued parameters are the nearest thing and are not an array type; a caller cannot + // bind a list to one parameter the way PostgreSQL allows. + ArrayTypes = false, + + // Query Notifications over Service Broker is exactly the thing a resident adapter could + // hold for free, and it is not wired yet. Declared false so the UI says "not available" + // rather than leaving a gap. Change Tracking and CDC are the same story. + ChangeNotification = false, + LogBasedCdc = false, + + SchemaDiscovery = true, + RowCountEstimates = true, + ReceiveModes = ["bulk", "incrementing", "timestamp", "timestamp+incrementing", "marker"], + + // TOP, which goes before the column list rather than after the query. OFFSET/FETCH exists + // too and is what paging uses, but it requires an ORDER BY — TOP is the form that works on + // a draft nobody has ordered yet. + LimitStyle = "top" + }; + + /// + /// What this principal may do IN THIS DATABASE, asked of the server rather than assumed. + /// fn_my_permissions accounts for role membership and for permissions granted at the server + /// level that reach down, which reading sys.database_permissions directly does not. + /// + protected override async Task> ProbePrivilegesAsync(DbConnection connection, + CancellationToken cancellationToken) + { + var privileges = new List(); + + try + { + using var command = connection.CreateCommand(); + command.CommandText = + "select permission_name from fn_my_permissions(null, 'DATABASE') order by permission_name"; + command.CommandTimeout = 10; + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + privileges.Add(reader.GetString(0)); + } + catch (Exception ex) + { + // A principal that cannot call fn_my_permissions is unusual but not broken — it just + // cannot tell us what it can do. Saying so beats failing the connection test over it. + Logger.LogDebug(ex, "Could not read permissions."); + privileges.Add($"(could not be read: {ex.Message})"); + } + + return privileges; + } + + protected override IEnumerable> ExtraStatusDetails() + { + yield return new KeyValuePair("sqlserver.database", _options.Database ?? ""); + yield return new KeyValuePair("sqlserver.schema", _options.Schema ?? "(login default)"); + yield return new KeyValuePair("sqlserver.encrypt", _options.Encrypt.ToString()); + yield return new KeyValuePair( + "sqlserver.trustServerCertificate", _options.TrustServerCertificate.ToString()); + } + + // ------------------------------------------------------------------ discovery + + /// + /// The sys catalog views rather than INFORMATION_SCHEMA. The standard views are defined to show + /// only what the caller has a permission on and cannot report an estimated row count at all; + /// sys.objects joined to the partition stats answers both in one query. + /// + protected override async Task> DiscoverAsync(DbConnection connection, + DiscoverRequest request, CancellationToken cancellationToken) + { + var schema = request.Schema; + var like = request.NameLike?.ToLowerInvariant(); + + return request.ObjectType switch + { + "procedure" or "function" => + await RoutinesAsync(connection, request, schema, like, cancellationToken), + "sequence" => await SequencesAsync(connection, request, schema, like, cancellationToken), + _ => await RelationsAsync(connection, request, schema, like, cancellationToken) + }; + } + + async Task> RelationsAsync(DbConnection connection, DiscoverRequest request, + string schema, string like, CancellationToken cancellationToken) + { + // U user table, V view. Nothing else in sys.objects is a thing an integration reads rows + // from. + var wanted = request.ObjectType == "view" ? "V" : "U"; + + var parameters = new Dictionary { ["wanted"] = wanted }; + var sql = new StringBuilder(@" + select s.name as [schema], o.name, o.type, + cast(ep.value as nvarchar(max)) as comment, + (select sum(p.rows) from sys.partitions p + where p.object_id = o.object_id and p.index_id in (0, 1)) as estimate + from sys.objects o + join sys.schemas s on s.schema_id = o.schema_id + left join sys.extended_properties ep + on ep.major_id = o.object_id and ep.minor_id = 0 and ep.name = 'MS_Description' + where o.type = @wanted and o.is_ms_shipped = 0"); + + if (schema != null) { sql.Append(" and s.name = @schema"); parameters["schema"] = schema; } + if (like != null) { sql.Append(" and charindex(@nameLike, lower(o.name)) > 0"); parameters["nameLike"] = like; } + + // OFFSET/FETCH needs an ORDER BY, which this has anyway — the list is meant to be stable + // between pages. + sql.Append(" order by s.name, o.name offset @skip rows fetch next @take rows only"); + parameters["skip"] = request.Skip; + parameters["take"] = request.Take; + + var objects = new List(); + using (var command = connection.CreateCommand()) + { + command.CommandText = sql.ToString(); + command.CommandTimeout = Options.CommandTimeoutSeconds; + AddParameters(command, parameters); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + objects.Add(new DbObject + { + Schema = reader.GetString(0), + Name = reader.GetString(1), + Type = reader.GetString(2).Trim() == "V" ? "view" : "table", + Comment = reader.IsDBNull(3) ? null : reader.GetString(3), + + // The heap or clustered index row count from the partition stats: maintained by + // the engine and cheap to read. Reported as an estimate everywhere it surfaces + // — the alternative is COUNT(*) on a stranger's table, which a menu should not + // do. Null for a view, which has no partitions of its own. + RowCount = !request.IncludeRowCounts || reader.IsDBNull(4) + ? null + : Math.Max(0, reader.GetInt64(4)) + }); + } + + if (request.IncludeColumns && objects.Count > 0) + await FillColumnsAsync(connection, objects, cancellationToken); + + return objects; + } + + async Task FillColumnsAsync(DbConnection connection, List objects, + CancellationToken cancellationToken) + { + // One query for the whole page, not one per object: a page of 200 tables would otherwise + // be 200 round trips, and against a remote database that is the difference between a + // screen that opens and one that times out. + var pairs = PairsOf(objects.Select(o => (o.Schema, o.Name))); + + using var command = connection.CreateCommand(); + command.CommandText = $@" + select s.name as [schema], o.name as [object], c.name as [column], + t.name as type_name, c.max_length, c.precision, c.scale, + c.is_nullable, c.column_id, + c.is_identity | c.is_computed | + (case when c.default_object_id <> 0 then 1 else 0 end) as generated, + (case when exists ( + select 1 + from sys.index_columns ic + join sys.indexes i + on i.object_id = ic.object_id and i.index_id = ic.index_id + where ic.object_id = c.object_id and ic.column_id = c.column_id + and i.is_primary_key = 1) then 1 else 0 end) as is_pk + from sys.columns c + join sys.objects o on o.object_id = c.object_id + join sys.schemas s on s.schema_id = o.schema_id + join sys.types t on t.user_type_id = c.user_type_id + and exists (select 1 from {pairs} as w(s, o) + where w.s = s.name collate database_default + and w.o = o.name collate database_default) + order by s.name, o.name, c.column_id"; + command.CommandTimeout = Options.CommandTimeoutSeconds; + + var byObject = objects.ToDictionary(o => $"{o.Schema}.{o.Name}"); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = $"{reader.GetString(0)}.{reader.GetString(1)}"; + if (!byObject.TryGetValue(key, out var target)) continue; + + var typeName = reader.GetString(3); + + // max_length is in BYTES, and an nvarchar stores two per character — so the number a + // schema was declared with is half of it. -1 is the max/blob form. + var maxLength = reader.GetInt16(4); + var wide = typeName.StartsWith("n", StringComparison.OrdinalIgnoreCase) + && typeName.IndexOf("char", StringComparison.OrdinalIgnoreCase) >= 0; + + target.Columns.Add(new DbColumn + { + Name = reader.GetString(2), + DbType = DescribeType(typeName, maxLength, reader.GetByte(5), reader.GetByte(6), wide), + ClrType = ClrTypeOf(typeName), + Nullable = reader.GetBoolean(7), + Ordinal = reader.GetInt32(8), + Generated = reader.GetInt32(9) != 0, + PrimaryKey = reader.GetInt32(10) != 0, + Length = maxLength < 0 ? null : wide ? maxLength / 2 : maxLength, + Precision = reader.GetByte(5), + Scale = reader.GetByte(6) + }); + } + } + + /// The type as a schema would declare it, which is what an operator recognises. + static string DescribeType(string typeName, short maxLength, byte precision, byte scale, bool wide) + { + var bare = typeName.ToLowerInvariant(); + + if (bare is "decimal" or "numeric") return $"{bare}({precision},{scale})"; + if (bare is "datetime2" or "time" or "datetimeoffset") return $"{bare}({scale})"; + + if (bare.EndsWith("char") || bare.EndsWith("binary")) + return maxLength < 0 ? $"{bare}(max)" : $"{bare}({(wide ? maxLength / 2 : maxLength)})"; + + return bare; + } + + async Task> RoutinesAsync(DbConnection connection, DiscoverRequest request, + string schema, string like, CancellationToken cancellationToken) + { + // P stored procedure; FN scalar, IF inline table-valued, TF multi-statement table-valued. + // All three function forms are worth listing — the table-valued ones are what a receive + // statement would select from. + var types = request.ObjectType == "procedure" + ? new[] { "P" } + : new[] { "FN", "IF", "TF" }; + + var parameters = new Dictionary(); + var placeholders = new List(); + for (var i = 0; i < types.Length; i++) + { + placeholders.Add($"@type{i}"); + parameters[$"type{i}"] = types[i]; + } + + var sql = new StringBuilder($@" + select s.name as [schema], o.name, o.type, + cast(ep.value as nvarchar(max)) as comment + from sys.objects o + join sys.schemas s on s.schema_id = o.schema_id + left join sys.extended_properties ep + on ep.major_id = o.object_id and ep.minor_id = 0 and ep.name = 'MS_Description' + where o.type in ({string.Join(",", placeholders)}) and o.is_ms_shipped = 0"); + + if (schema != null) { sql.Append(" and s.name = @schema"); parameters["schema"] = schema; } + if (like != null) { sql.Append(" and charindex(@nameLike, lower(o.name)) > 0"); parameters["nameLike"] = like; } + + sql.Append(" order by s.name, o.name offset @skip rows fetch next @take rows only"); + parameters["skip"] = request.Skip; + parameters["take"] = request.Take; + + var objects = new List(); + using (var command = connection.CreateCommand()) + { + command.CommandText = sql.ToString(); + command.CommandTimeout = Options.CommandTimeoutSeconds; + AddParameters(command, parameters); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + objects.Add(new DbObject + { + Schema = reader.GetString(0), + Name = reader.GetString(1), + Type = reader.GetString(2).Trim() == "P" ? "procedure" : "function", + Comment = reader.IsDBNull(3) ? null : reader.GetString(3) + }); + } + + if (objects.Count > 0) await FillParametersAsync(connection, objects, cancellationToken); + return objects; + } + + async Task FillParametersAsync(DbConnection connection, List routines, + CancellationToken cancellationToken) + { + var pairs = PairsOf(routines.Select(r => (r.Schema, r.Name))); + + using var command = connection.CreateCommand(); + command.CommandText = $@" + select s.name as [schema], o.name as [object], p.name as parameter, + t.name as type_name, p.is_output, p.parameter_id, p.max_length, p.precision, p.scale + from sys.parameters p + join sys.objects o on o.object_id = p.object_id + join sys.schemas s on s.schema_id = o.schema_id + join sys.types t on t.user_type_id = p.user_type_id + where exists (select 1 from {pairs} as w(s, o) + where w.s = s.name collate database_default + and w.o = o.name collate database_default) + order by s.name, o.name, p.parameter_id"; + command.CommandTimeout = Options.CommandTimeoutSeconds; + + var byRoutine = routines.ToDictionary(r => $"{r.Schema}.{r.Name}"); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = $"{reader.GetString(0)}.{reader.GetString(1)}"; + if (!byRoutine.TryGetValue(key, out var target)) continue; + + // parameter_id 0 is a scalar function's return value, and it has no name. + var ordinal = reader.GetInt32(5); + var name = reader.IsDBNull(2) || reader.GetString(2).Length == 0 + ? (ordinal == 0 ? "(returns)" : $"@p{ordinal}") + : reader.GetString(2).TrimStart('@'); + + var typeName = reader.GetString(3); + var wide = typeName.StartsWith("n", StringComparison.OrdinalIgnoreCase) + && typeName.IndexOf("char", StringComparison.OrdinalIgnoreCase) >= 0; + + target.Parameters.Add(new DbRoutineParameter + { + Name = name, + DbType = DescribeType(typeName, reader.GetInt16(6), reader.GetByte(7), reader.GetByte(8), wide), + Direction = ordinal == 0 ? "ReturnValue" : reader.GetBoolean(4) ? "Out" : "In", + Ordinal = ordinal + }); + } + } + + async Task> SequencesAsync(DbConnection connection, DiscoverRequest request, + string schema, string like, CancellationToken cancellationToken) + { + var parameters = new Dictionary(); + var sql = new StringBuilder(@" + select s.name as [schema], q.name, cast(q.current_value as bigint) as current_value + from sys.sequences q + join sys.schemas s on s.schema_id = q.schema_id + where q.is_ms_shipped = 0"); + + if (schema != null) { sql.Append(" and s.name = @schema"); parameters["schema"] = schema; } + if (like != null) { sql.Append(" and charindex(@nameLike, lower(q.name)) > 0"); parameters["nameLike"] = like; } + + sql.Append(" order by s.name, q.name offset @skip rows fetch next @take rows only"); + parameters["skip"] = request.Skip; + parameters["take"] = request.Take; + + var objects = new List(); + using var command = connection.CreateCommand(); + command.CommandText = sql.ToString(); + command.CommandTimeout = Options.CommandTimeoutSeconds; + AddParameters(command, parameters); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + objects.Add(new DbObject + { + Schema = reader.GetString(0), + Name = reader.GetString(1), + Type = "sequence", + + // Null until the sequence has been used at all, which is a meaningful answer of its + // own: nothing has drawn from this counter yet. + RowCount = reader.IsDBNull(2) ? null : reader.GetInt64(2) + }); + + return objects; + } + + /// + /// The page's (schema, object) pairs as a table to join against. + /// + /// SQL Server has no row constructor in IN — where (a, b) in ((..),(..)) is PostgreSQL + /// and MySQL syntax and is a parse error here — so the pairs become a VALUES table and the + /// filter becomes an EXISTS against it. The collation is forced to the database's own because + /// a VALUES literal takes the server default, and on an instance whose default differs from + /// the database's the comparison fails outright rather than merely matching oddly. + /// + static string PairsOf(IEnumerable<(string Schema, string Name)> objects) => + "(values " + string.Join(",", objects.Select(o => $"({Quote(o.Schema)},{Quote(o.Name)})")) + ")"; + + /// + /// An N-prefixed literal for the pairs above. Doubling the quote is SQL Server's own + /// escape, and these are identifiers this connection just read back from the catalog — quoted + /// anyway, because "it cannot contain a quote" is the kind of assumption that survives right up + /// until a table is named oddly. + /// + static string Quote(string value) => "N'" + (value ?? "").Replace("'", "''") + "'"; + + /// + /// What a value of this column arrives as in a result row. Coarse on purpose — it tells a + /// mapper author whether to expect a string or a number, and is not trying to be a type system. + /// + static string ClrTypeOf(string typeName) => typeName.ToLowerInvariant() switch + { + "bit" => "bool", + "tinyint" or "smallint" or "int" => "int", + "bigint" => "long", + "decimal" or "numeric" or "money" or "smallmoney" => "decimal", + "float" or "real" => "double", + "date" or "datetime" or "datetime2" or "smalldatetime" => "DateTime", + "datetimeoffset" => "DateTimeOffset", + "time" => "TimeSpan", + "uniqueidentifier" => "Guid", + "binary" or "varbinary" or "image" or "timestamp" or "rowversion" => "byte[]", + "xml" => "string", + _ => "string" + }; + + void AddParameters(DbCommand command, Dictionary parameters) + { + foreach (var kv in parameters) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = kv.Key; + parameter.Value = kv.Value ?? DBNull.Value; + command.Parameters.Add(parameter); + } + } +} diff --git a/SW.Bitween.Adapters.Db.SqlServer/SqlServerOptions.cs b/SW.Bitween.Adapters.Db.SqlServer/SqlServerOptions.cs new file mode 100644 index 00000000..33b6fe09 --- /dev/null +++ b/SW.Bitween.Adapters.Db.SqlServer/SqlServerOptions.cs @@ -0,0 +1,80 @@ +using SW.Bitween.Adapters; + +namespace SW.Bitween.Adapters.Db.SqlServer; + +/// +/// Every setting here arrives as a DataSource property, bound by name, and the form an operator +/// fills in is generated from these attributes — so a field added here appears in Bitween with no +/// front-end change. +/// +/// The hints spend their weight on encryption, because SQL Server is the engine where the default +/// changed underneath everybody: the modern driver encrypts by default and then refuses a +/// self-signed certificate, which is what most on-premises instances present. +/// +[AdapterSettings( + Kind = "Relational", + Label = "SQL Server", + Description = "A Microsoft SQL Server or Azure SQL database, held open with a pooled " + + "connection so statements, procedures and polling receivers do not pay a " + + "connect on every message.")] +public class SqlServerOptions : DbOptionsBase +{ + [AdapterSetting(Required = true, Hint = + "Host name, IP, or host\\instance for a named instance. No tcp: prefix — set Port instead.")] + public string Host { get; set; } = "localhost"; + + [AdapterSetting(Default = "1433", Hint = + "Ignored when Host names an instance: a named instance is resolved through the SQL Server " + + "Browser rather than by port.")] + public int Port { get; set; } = 1433; + + [AdapterSetting(Required = true, Hint = "The database to connect to, not the server.")] + public string Database { get; set; } + + [AdapterSetting(Required = true, Hint = + "A SQL login, or DOMAIN\\user for Windows authentication where the host supports it.")] + public string UserName { get; set; } + + [AdapterSetting(Secret = true, Required = true)] + public string Password { get; set; } + + /// + /// The default schema for unqualified names is a property of the LOGIN, not of the connection + /// string — there is no equivalent of PostgreSQL's search_path to set here. This runs a + /// per-connection default instead, which is why it is worded as what it does rather than as a + /// setting name. + /// + [AdapterSetting(Hint = + "Run as this schema, so an unqualified table name in a statement resolves against it. " + + "Leave empty to use the login's own default, which is usually dbo. Only takes effect if " + + "the login can impersonate the schema's owner; when it cannot, the connection still " + + "works and unqualified names keep resolving as before.")] + public string Schema { get; set; } + + [AdapterSetting(Default = "true", AllowedValues = new[] { "true", "false" }, Hint = + "Encrypt the connection. On by default in the modern driver, which is a change from the " + + "old one — and the reason an instance that worked for years starts failing on upgrade.")] + public bool Encrypt { get; set; } = true; + + [AdapterSetting(Default = "false", AllowedValues = new[] { "true", "false" }, Hint = + "Accept the server's certificate without validating it. Needed for the usual on-premises " + + "instance with a self-signed certificate — and it means an attacker between here and the " + + "server could present their own. Prefer installing the certificate; use this knowingly.")] + public bool TrustServerCertificate { get; set; } + + [AdapterSetting(Default = "Bitween", Hint = + "What this connection calls itself in sys.dm_exec_sessions. Worth keeping distinctive: it " + + "is how a DBA works out which of the connections on their server is yours.")] + public string ApplicationName { get; set; } = "Bitween"; + + [AdapterSetting(Default = "false", AllowedValues = new[] { "true", "false" }, Hint = + "Use MultipleActiveResultSets. Off unless something needs it: it lets one connection hold " + + "several open readers, and the cost is that the connection cannot be reset cleanly " + + "between uses, which is the opposite of what a shared pool wants.")] + public bool MultipleActiveResultSets { get; set; } + + [AdapterSetting(Default = "false", AllowedValues = new[] { "true", "false" }, Hint = + "Read committed snapshot for this connection's transactions, so readers do not block " + + "writers. Only has an effect where the database has snapshot isolation enabled.")] + public bool SnapshotIsolation { get; set; } +} diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs index b73ab514..06ac374c 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs @@ -42,7 +42,7 @@ await requestContext.EnsurePermission(dbContext, + "something to a Relational one."); await EnsureNameIsFree(dbContext, dataSourceId, model.Name, existingId: null); - await EnsureSqlIsValid(validator, dataSourceId, model.Sql); + var checkedSql = await EnsureSqlIsValid(validator, dataSourceId, model.Sql); var entity = new DataSourceStatement { @@ -58,7 +58,10 @@ await requestContext.EnsurePermission(dbContext, dbContext.Add(entity); await dbContext.SaveChangesAsync(); - return entity.Id; + // The id, and whether the SQL was actually checked. Saying so matters because validation + // is best-effort: when the adapter is not running there is nobody to ask, and a save that + // reported nothing would look exactly like one that had been verified. + return new { Id = entity.Id, Checked = checkedSql }; } /// @@ -69,12 +72,15 @@ await requestContext.EnsurePermission(dbContext, /// connection, and blocking someone from saving a fix because the thing they are fixing it /// for is broken would be exactly backwards. /// - internal static async Task EnsureSqlIsValid( + /// True when the database actually looked at it; false when nobody could be asked. + internal static async Task EnsureSqlIsValid( SW.Bitween.Services.DataSources.StatementValidator validator, int dataSourceId, string sql) { var result = await validator.ValidateAsync(dataSourceId, sql); if (result.Checked && !result.Ok) throw new SWException($"The database will not accept this statement. {result.Error}"); + + return result.Checked; } /// diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs index 9ca659e1..403da43b 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs @@ -43,8 +43,11 @@ public async Task Handle(int key, DataSourceStatementUpdate model) // Only when it actually changed: re-checking untouched SQL would refuse a rename, or a // change of owner, because of a table someone dropped last week — a fault worth surfacing // but not here, and not as a block on an unrelated edit. + // True when the SQL did not change: there was nothing to check, which is not the same as + // "could not check" and should not warn as though it were. + var checkedSql = true; if (!string.Equals(entity.Sql, model.Sql, System.StringComparison.Ordinal)) - await Create.EnsureSqlIsValid(validator, entity.DataSourceId, model.Sql); + checkedSql = await Create.EnsureSqlIsValid(validator, entity.DataSourceId, model.Sql); entity.Name = model.Name.Trim(); entity.Sql = model.Sql; @@ -55,7 +58,7 @@ public async Task Handle(int key, DataSourceStatementUpdate model) entity.Inactive = model.Inactive; await dbContext.SaveChangesAsync(); - return entity.Id; + return new { Id = entity.Id, Checked = checkedSql }; } private class Validate : AbstractValidator diff --git a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs index 0f8c3fda..ffd74b51 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs @@ -322,6 +322,14 @@ await AdapterInstaller.InstallAsync(cloudFiles, "SW.Bitween.Adapters.Db.PostgreSql", BusAdapters.PostgreSql, "SW.Bitween.Adapters.Db.PostgreSql.dll", new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.Adapters.Db.MySql", BusAdapters.MySql, + "SW.Bitween.Adapters.Db.MySql.dll", + new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.Adapters.Db.SqlServer", BusAdapters.SqlServer, + "SW.Bitween.Adapters.Db.SqlServer.dll", + new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); } await App.StartAsync(); diff --git a/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs b/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs index 231d0653..84957069 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs @@ -12,4 +12,6 @@ public static class BusAdapters /// public const string Oracle = "bitween.db.oracle"; public const string PostgreSql = "bitween.db.postgresql"; + public const string MySql = "bitween.db.mysql"; + public const string SqlServer = "bitween.db.sqlserver"; } diff --git a/SW.Bitween.IntegrationTests/Fixtures/MySqlDbFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/MySqlDbFixture.cs new file mode 100644 index 00000000..faab1d56 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Fixtures/MySqlDbFixture.cs @@ -0,0 +1,125 @@ +using System; +using System.Threading.Tasks; +using MySqlConnector; +using Testcontainers.MySql; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Fixtures; + +/// +/// A MySQL for the adapter tests to point at — deliberately its own container rather than the one +/// runs for the application database. +/// +/// A class fixture, not on the test class: xUnit builds a new instance +/// of a test class per test method, so a container started there is a container per test. +/// +/// The seed mirrors the PostgreSQL one object for object, because the point of having both is that +/// the shared core behaves identically and the differences show up where they are real — here, a +/// procedure that returns rows by SELECTing, which is the thing PostgreSQL cannot do. +/// +public class MySqlDbFixture : IAsyncLifetime +{ + public const string Table = "bitween_orders"; + + MySqlContainer _container; + + /// Set when Docker is unavailable, so the tests skip rather than fail. + public string Unavailable { get; private set; } + + public string Host => _container.Hostname; + public int Port => _container.GetMappedPublicPort(3306); + public string Database => "bitween_adapter"; + public string User => "bitween"; + public string Password => "bitween_pw"; + + public async Task InitializeAsync() + { + try + { + _container = new MySqlBuilder() + .WithImage("mysql:8.4") + .WithDatabase(Database) + .WithUsername(User) + .WithPassword(Password) + + // Creating a FUNCTION requires SUPER while binary logging is on, because a + // non-deterministic one would make the binary log unsafe to replay. Our user is + // not SUPER and should not be, so the server is told to trust function creators — + // which is the switch a DBA sets for exactly this, and is why the adapter's own + // capability list does not promise that creating routines is something it can do. + .WithCommand("--log-bin-trust-function-creators=1") + .Build(); + + await _container.StartAsync(); + await SeedAsync(); + } + catch (Exception ex) + { + Unavailable = ex.Message; + } + } + + public async Task DisposeAsync() + { + if (_container != null) await _container.DisposeAsync(); + } + + public string AdminConnectionString => + $"Server={Host};Port={Port};Database={Database};User ID={User};Password={Password};" + + "AllowUserVariables=true;AllowPublicKeyRetrieval=true"; + + async Task SeedAsync() + { + await using var connection = new MySqlConnection(AdminConnectionString); + await connection.OpenAsync(); + + await ExecuteAsync(connection, $@" + create table {Table} ( + id int primary key, + customer varchar(50), + amount decimal(10,2), + created_at timestamp not null default current_timestamp, + processed tinyint(1) not null default 0 + ) comment 'Customer orders'"); + + for (var i = 1; i <= 25; i++) + await ExecuteAsync(connection, + $"insert into {Table} (id, customer, amount) " + + $"values ({i}, '{(i % 2 == 0 ? "acme" : "globex")}', {i * 10}.50)"); + + // A procedure that returns rows simply by SELECTing — no cursor to declare, nothing to + // bind. This is the capability PostgreSQL declares false and MySQL declares true, and the + // reason the two adapters are worth testing against the same shape of schema. + await ExecuteAsync(connection, $@" + create procedure orders_by_customer(in p_customer varchar(50)) + begin + select * from {Table} where customer = p_customer order by id; + end"); + + // A procedure that returns a value through an OUT parameter rather than a result set. + await ExecuteAsync(connection, $@" + create procedure count_orders(in p_customer varchar(50), out p_total int) + begin + select count(*) into p_total from {Table} where customer = p_customer; + end"); + + // And a scalar function, which is a different object type in the catalog. + await ExecuteAsync(connection, $@" + create function total_for(p_customer varchar(50)) + returns decimal(12,2) + deterministic + reads sql data + begin + declare v_total decimal(12,2); + select coalesce(sum(amount), 0) into v_total from {Table} where customer = p_customer; + return v_total; + end"); + } + + static async Task ExecuteAsync(MySqlConnection connection, string sql) + { + await using var command = connection.CreateCommand(); + command.CommandText = sql; + await command.ExecuteNonQueryAsync(); + } +} diff --git a/SW.Bitween.IntegrationTests/Fixtures/SqlServerDbFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/SqlServerDbFixture.cs new file mode 100644 index 00000000..75cb4928 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Fixtures/SqlServerDbFixture.cs @@ -0,0 +1,184 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using DotNet.Testcontainers.Builders; +using Testcontainers.MsSql; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Fixtures; + +/// +/// A SQL Server for the adapter tests to point at — deliberately its own container rather than the +/// one runs for the application database. +/// +/// A class fixture, not on the test class: xUnit builds a new instance +/// of a test class per test method, so a container started there is a container per test. +/// +/// The image is the 2022 developer edition, which is what Testcontainers defaults to and what runs +/// on both x64 and, through emulation, Apple silicon. It is around 1.5 GB — an order of magnitude +/// less than Oracle's, so unlike those these run on every pass rather than nightly. +/// +public class SqlServerDbFixture : IAsyncLifetime +{ + public const string Table = "bitween_orders"; + + MsSqlContainer _container; + + /// Set when Docker is unavailable, so the tests skip rather than fail. + public string Unavailable { get; private set; } + + public string Host => _container.Hostname; + public int Port => _container.GetMappedPublicPort(1433); + + /// + /// A database of our own rather than the container's master. Discovery filters by schema and + /// master is full of the engine's own objects, so a test asserting on what it can see would be + /// asserting about Microsoft's schema as much as ours. + /// + public string Database => "bitween_adapter"; + + public string User => "sa"; + public string Password => "yourStrong(!)Password"; + + public async Task InitializeAsync() + { + try + { + _container = new MsSqlBuilder() + // Pinned, and not to Testcontainers' default. That default is 2019, which has no + // arm64 image and no emulation path — it exits the moment it starts on an Apple + // silicon machine, which surfaces as "container is not running" rather than as + // anything about the architecture. 2022 is the first release Microsoft publishes + // for arm64, and it runs on both. + .WithImage("mcr.microsoft.com/mssql/server:2022-CU14-ubuntu-22.04") + .WithPassword(Password) + + // Wait on the PORT, not on the built-in sqlcmd probe. SQL Server listens well + // before it will accept a login, and under emulation on Apple silicon the gap is + // long enough that the default readiness check gives up and the container is torn + // down — which surfaces as "container is not running" rather than as a timeout. + .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(1433)) + .Build(); + + await _container.StartAsync(); + await WaitForLoginsAsync(); + await SeedAsync(); + } + catch (Exception ex) + { + Unavailable = ex.Message; + } + } + + public async Task DisposeAsync() + { + if (_container != null) await _container.DisposeAsync(); + } + + /// + /// TrustServerCertificate because the container presents a self-signed certificate, which is + /// the same reason the adapter offers the setting at all. + /// + public string AdminConnectionString => + $"Server={Host},{Port};Database={Database};User ID={User};Password={Password};" + + "Encrypt=True;TrustServerCertificate=True"; + + string MasterConnectionString => + $"Server={Host},{Port};Database=master;User ID={User};Password={Password};" + + "Encrypt=True;TrustServerCertificate=True"; + + /// + /// Polls until a login succeeds. The port opening only means the process is up; recovery of + /// master, msdb and tempdb runs after that, and a connection during it is refused with "not + /// currently available". Two minutes, because emulation is slow and a flaky skip reads as a + /// broken adapter. + /// + async Task WaitForLoginsAsync() + { + var deadline = DateTime.UtcNow.AddMinutes(2); + while (true) + { + try + { + await using var probe = new SqlConnection(MasterConnectionString); + await probe.OpenAsync(); + return; + } + catch when (DateTime.UtcNow < deadline) + { + await Task.Delay(TimeSpan.FromSeconds(3)); + } + } + } + + async Task SeedAsync() + { + await using (var master = new SqlConnection(MasterConnectionString)) + { + await master.OpenAsync(); + await ExecuteAsync(master, $"create database [{Database}]"); + } + + await using var connection = new SqlConnection(AdminConnectionString); + await connection.OpenAsync(); + + // A schema of its own, so the discovery tests can prove the schema filter does something — + // everything would otherwise be dbo, where a filter that did nothing would still pass. + await ExecuteAsync(connection, "create schema sales"); + + await ExecuteAsync(connection, $@" + create table sales.{Table} ( + id int not null primary key, + customer nvarchar(50) null, + amount decimal(10,2) null, + created_at datetime2(3) not null constraint df_created default sysutcdatetime(), + processed bit not null constraint df_processed default 0 + )"); + + await ExecuteAsync(connection, $@" + exec sys.sp_addextendedproperty + @name = N'MS_Description', @value = N'Customer orders', + @level0type = N'SCHEMA', @level0name = N'sales', + @level1type = N'TABLE', @level1name = N'{Table}'"); + + await ExecuteAsync(connection, "create sequence sales.bitween_order_seq as bigint start with 1000"); + + for (var i = 1; i <= 25; i++) + await ExecuteAsync(connection, + $"insert into sales.{Table} (id, customer, amount) " + + $"values ({i}, '{(i % 2 == 0 ? "acme" : "globex")}', {i * 10}.50)"); + + // A procedure that returns rows simply by SELECTing, like MySQL and unlike PostgreSQL. + await ExecuteAsync(connection, $@" + create procedure sales.orders_by_customer @p_customer nvarchar(50) + as + begin + set nocount on; + select * from sales.{Table} where customer = @p_customer order by id; + end"); + + // And one that answers through an OUT parameter instead. + await ExecuteAsync(connection, $@" + create procedure sales.count_orders @p_customer nvarchar(50), @p_total int output + as + begin + set nocount on; + select @p_total = count(*) from sales.{Table} where customer = @p_customer; + end"); + + // An inline table-valued function — the SQL Server shape a receive statement would select + // from, and a different object type in the catalog from a scalar one. + await ExecuteAsync(connection, $@" + create function sales.orders_for(@p_customer nvarchar(50)) + returns table + as + return (select * from sales.{Table} where customer = @p_customer)"); + } + + static async Task ExecuteAsync(SqlConnection connection, string sql) + { + await using var command = connection.CreateCommand(); + command.CommandText = sql; + await command.ExecuteNonQueryAsync(); + } +} diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index 41d03d76..385c44d2 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -26,8 +26,14 @@ + + + + + @@ -58,6 +64,8 @@ + + diff --git a/SW.Bitween.IntegrationTests/Tests/DataSourceStatementTests.cs b/SW.Bitween.IntegrationTests/Tests/DataSourceStatementTests.cs index c9e1aaea..cad79b0d 100644 --- a/SW.Bitween.IntegrationTests/Tests/DataSourceStatementTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/DataSourceStatementTests.cs @@ -259,8 +259,13 @@ async Task CreateStatementAsync(int dataSourceId, string name, string sql) var handler = ActivatorUtilities.CreateInstance( scope.ServiceProvider); - return (int)await handler.Handle( + // The handler answers with { Id, Checked } — the id, and whether the database actually + // looked at the SQL. Read through the property rather than cast, because the shape is an + // anonymous type and a cast to int is what this used to do. + var created = await handler.Handle( new DataSourceStatementCreate { DataSourceId = dataSourceId, Name = name, Sql = sql }); + + return (int)created.GetType().GetProperty("Id")!.GetValue(created)!; } async Task UpdateAsync(int id, DataSourceStatementUpdate model) diff --git a/SW.Bitween.IntegrationTests/Tests/MySqlAdapterTests.cs b/SW.Bitween.IntegrationTests/Tests/MySqlAdapterTests.cs new file mode 100644 index 00000000..fd86bd88 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/MySqlAdapterTests.cs @@ -0,0 +1,540 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Serverless.Resident; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The MySQL data source provider, against a real MySQL. +/// +/// The same suite as PostgreSQL's and Oracle's, which is the point: three engines run on one core, +/// so the parts that are shared should behave identically and the parts that differ should differ +/// VISIBLY — in the capability list, and in how a routine returns rows. Everything here goes over +/// the resident transport into a real adapter process and out to a real database. +/// +[Collection("Bitween")] +public class MySqlAdapterTests : IClassFixture +{ + readonly BitweenFixture _fixture; + readonly MySqlDbFixture _mysql; + + public MySqlAdapterTests(BitweenFixture fixture, MySqlDbFixture mysql) + { + _fixture = fixture; + _mysql = mysql; + } + + static readonly SemaphoreSlim Gate = new(1, 1); + static int _dataSourceId; + + IResidentAdapterHost Host => _fixture.App.Services.GetRequiredService(); + + async Task AdapterAsync() + { + Skip.If(_mysql.Unavailable != null, $"MySQL is not available: {_mysql.Unavailable}"); + + await Gate.WaitAsync(); + try + { + if (_dataSourceId == 0) + { + _dataSourceId = await CreateDataSourceAsync(); + await StartAdapterAsync(); + } + } + finally + { + Gate.Release(); + } + + return Host.Get(BusAdapters.MySql, _dataSourceId.ToString()) + ?? throw new InvalidOperationException("The MySQL adapter is not running."); + } + + // ---------------------------------------------------------------- configuration + + [SkippableFact] + public async Task Connection_test_reports_each_stage_and_prepares_every_statement() + { + var adapter = await AdapterAsync(); + var result = await adapter.InvokeAsync("TestConnection", timeoutSeconds: 60); + + Assert.True(result.Value("ok"), result.ToString()); + + var steps = result["steps"]!.Select(s => s.Value("step")).ToList(); + Assert.Contains("connect", steps); + Assert.Contains("authenticate", steps); + Assert.Contains("privileges", steps); + Assert.Contains("statement:ordersForCustomer", steps); + Assert.All(result["steps"]!, s => Assert.True(s.Value("ok"), s.ToString())); + } + + /// + /// Where MySQL is honestly different from the other two. A procedure returns rows by SELECTing, + /// which PostgreSQL cannot do — but there is no MERGE, no RETURNING, no array type and no + /// sequence, and claiming any of them would have a statement written against it fail on a real + /// server. + /// + [SkippableFact] + public async Task Describe_reports_the_engine_and_where_it_differs() + { + var adapter = await AdapterAsync(); + var described = await adapter.InvokeAsync("Describe", timeoutSeconds: 60); + + Assert.Equal("MySQL", described.Value("engine")); + Assert.False(string.IsNullOrWhiteSpace(described.Value("serverVersion"))); + + // The capability PostgreSQL declares false and this one declares true. + Assert.True(described.Value("storedProcedures")); + Assert.True(described.Value("procedureResultSets")); + + // And the three it does not have. + Assert.False(described.Value("merge")); + Assert.False(described.Value("returning")); + Assert.False(described.Value("arrayTypes")); + + var objects = described["supportedObjects"]!.Select(o => o.Value()).ToList(); + Assert.Contains("procedure", objects); + Assert.Contains("function", objects); + // No sequences in MySQL — AUTO_INCREMENT belongs to a column, not to an object. + Assert.DoesNotContain("sequence", objects); + + // READ UNCOMMITTED does something here, unlike PostgreSQL where it is silently promoted. + var isolation = described["isolationLevels"]!.Select(i => i.Value()).ToList(); + Assert.Contains("ReadUncommitted", isolation); + + Assert.False(described.Value("logBasedCdc")); + Assert.False(described.Value("changeNotification")); + + Assert.NotEmpty(described["privileges"]!); + } + + // ---------------------------------------------------------------- discovery + + [SkippableFact] + public async Task Discover_finds_the_table_with_its_columns_and_key() + { + var adapter = await AdapterAsync(); + var result = await adapter.InvokeAsync("Discover", new + { + objectType = "table", + nameLike = MySqlDbFixture.Table, + includeColumns = true, + includeRowCounts = true + }, timeoutSeconds: 60); + + var table = result["objects"]!.Single(o => o.Value("name") == MySqlDbFixture.Table); + + // A schema IS a database in MySQL, and this is what that means in practice. + Assert.Equal(_mysql.Database, table.Value("schema")); + Assert.Equal("Customer orders", table.Value("comment")); + + var columns = table["columns"]!.ToList(); + Assert.Equal("id", columns[0].Value("name")); + Assert.True(columns[0].Value("primaryKey")); + Assert.False(columns[0].Value("nullable")); + + // tinyint(1) IS the boolean type in MySQL, and the driver returns one — so reporting "int" + // would mislead whoever writes the mapper. + var processed = columns.Single(c => c.Value("name") == "processed"); + Assert.Equal("bool", processed.Value("clrType")); + + // A CURRENT_TIMESTAMP default reports as DEFAULT_GENERATED in `extra`, which is a value the + // database fills in — the only distinction an insert cares about. + var created = columns.Single(c => c.Value("name") == "created_at"); + Assert.True(created.Value("generated")); + + var amount = columns.Single(c => c.Value("name") == "amount"); + Assert.Equal("decimal", amount.Value("clrType")); + Assert.Equal(10, amount.Value("precision")); + Assert.Equal(2, amount.Value("scale")); + } + + [SkippableFact] + public async Task Discover_finds_procedures_and_functions_with_their_parameters() + { + var adapter = await AdapterAsync(); + + var procedures = await adapter.InvokeAsync("Discover", + new { objectType = "procedure" }, timeoutSeconds: 60); + + var counting = procedures["objects"]!.Single(o => o.Value("name") == "count_orders"); + var parameters = counting["parameters"]!.ToList(); + + Assert.Equal("p_customer", parameters[0].Value("name")); + Assert.Equal("In", parameters[0].Value("direction")); + Assert.Equal("p_total", parameters[1].Value("name")); + Assert.Equal("Out", parameters[1].Value("direction")); + + var functions = await adapter.InvokeAsync("Discover", + new { objectType = "function" }, timeoutSeconds: 60); + + var total = functions["objects"]!.Single(o => o.Value("name") == "total_for"); + + // A function's return type comes from dtd_identifier and is listed once, not twice. + var returns = total["parameters"]!.Where(p => p.Value("direction") == "ReturnValue").ToList(); + Assert.Single(returns); + } + + [SkippableFact] + public async Task Discover_filters_by_name_against_the_database_not_the_page() + { + var adapter = await AdapterAsync(); + + var found = await adapter.InvokeAsync("Discover", + new { objectType = "table", nameLike = "ORDERS" }, timeoutSeconds: 60); + Assert.NotEmpty(found["objects"]!); + + var nothing = await adapter.InvokeAsync("Discover", + new { objectType = "table", nameLike = "no_such_table_anywhere" }, timeoutSeconds: 60); + Assert.Empty(nothing["objects"]!); + } + + // ---------------------------------------------------------------- running + + [SkippableFact] + public async Task Query_returns_rows_for_a_named_statement() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Query", new + { + name = "ordersForCustomer", + parameters = new Dictionary { ["customer"] = "acme" } + }, timeoutSeconds: 60); + + var rows = result["rows"]!.ToList(); + Assert.NotEmpty(rows); + Assert.All(rows, r => Assert.Equal("acme", r.Value("customer"))); + } + + /// + /// The difference that matters: CALL hands back a result set with nothing declared and nothing + /// bound. On PostgreSQL this same shape needs a set-returning function queried with SELECT, and + /// on Oracle an explicit REF CURSOR. + /// + [SkippableFact] + public async Task Call_returns_rows_straight_from_a_procedure() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Call", new + { + name = "ordersByCustomerProc", + parameters = new Dictionary { ["p_customer"] = "globex" } + }, timeoutSeconds: 60); + + var rows = result["rows"]!.ToList(); + Assert.NotEmpty(rows); + Assert.All(rows, r => Assert.Equal("globex", r.Value("customer"))); + } + + [SkippableFact] + public async Task Execute_reports_what_it_changed() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Execute", new + { + name = "insertOrder", + parameters = new Dictionary + { ["id"] = 801, ["customer"] = "inserted", ["amount"] = 12.34 } + }, timeoutSeconds: 60); + + Assert.Equal(1, result.Value("affectedRows")); + + var back = await adapter.InvokeAsync("Query", new + { + name = "ordersForCustomer", + parameters = new Dictionary { ["customer"] = "inserted" } + }, timeoutSeconds: 60); + + Assert.Single(back["rows"]!); + } + + /// + /// A batch is one transaction: either every statement in it happened or none did. Proven by + /// making the second one fail — a duplicate primary key — and then looking for the first. + /// + [SkippableFact] + public async Task A_failed_batch_rolls_back_what_came_before_it() + { + var adapter = await AdapterAsync(); + + await Assert.ThrowsAnyAsync(() => adapter.InvokeAsync("Batch", new + { + statements = new object[] + { + new + { + name = "insertOrder", + parameters = new Dictionary + { ["id"] = 850, ["customer"] = "rolled-back", ["amount"] = 1 } + }, + new + { + // id 1 is seeded, so this violates the primary key. + name = "insertOrder", + parameters = new Dictionary + { ["id"] = 1, ["customer"] = "rolled-back", ["amount"] = 1 } + } + } + }, timeoutSeconds: 60)); + + var back = await adapter.InvokeAsync("Query", new + { + name = "ordersForCustomer", + parameters = new Dictionary { ["customer"] = "rolled-back" } + }, timeoutSeconds: 60); + + Assert.Empty(back["rows"]!); + } + + /// + /// Ad-hoc SQL is refused unless the data source allows it, whatever the engine. The mapper is a + /// template over message content — if it can emit SQL text, every Xchange is an injection + /// vector into the customer's database. + /// + [SkippableFact] + public async Task Sql_sent_with_a_message_is_refused() + { + var adapter = await AdapterAsync(); + + var error = await Assert.ThrowsAnyAsync(() => + adapter.InvokeAsync("Query", + new { sql = $"select * from {MySqlDbFixture.Table}" }, timeoutSeconds: 60)); + + Assert.Contains("does not allow ad-hoc SQL", error.Message); + } + + // ---------------------------------------------------------------- receiving + + [SkippableFact] + public async Task The_receiver_advances_a_cursor_that_survives_a_restart() + { + var adapter = await AdapterAsync(); + var me = Subscription(1); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: me); + var first = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: me); + + Assert.Equal(5, first.Count); + + foreach (var id in first) + { + var file = await adapter.InvokeAsync("GetFile", id, timeoutSeconds: 60, + properties: me); + var data = file.Value("data") ?? file.Value("Data"); + Assert.False(string.IsNullOrWhiteSpace(data)); + + await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 60, properties: me); + } + + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: me); + + Assert.Equal("5", await CursorAsync("receive.cursor.1")); + + var restarted = await Host.RestartAsync(BusAdapters.MySql, _dataSourceId.ToString(), + drain: false); + + await restarted.InvokeAsync("Initialize", timeoutSeconds: 60, properties: me); + var second = await restarted.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: me); + + Assert.Equal(new[] { 6, 7, 8, 9, 10 }, second.Select(KeyOf).OrderBy(i => i)); + } + + [SkippableFact] + public async Task Two_subscriptions_on_one_data_source_do_not_share_a_cursor() + { + var adapter = await AdapterAsync(); + + var first = Subscription(101); + var second = Subscription(202); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: first); + var forFirst = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: first); + foreach (var id in forFirst) + await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 60, properties: first); + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: first); + + Assert.NotEmpty(forFirst); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: second); + var forSecond = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: second); + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: second); + + Assert.Equal( + forFirst.Select(KeyOf).OrderBy(k => k), + forSecond.Select(KeyOf).OrderBy(k => k)); + } + + [SkippableFact] + public async Task Mark_processed_runs_for_each_accepted_row() + { + var adapter = await AdapterAsync(); + var me = Subscription(2); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: me); + var listed = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: me); + + foreach (var id in listed) + await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 60, properties: me); + + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: me); + + var processed = await adapter.InvokeAsync("Query", new { name = "processedOrders" }, + timeoutSeconds: 60); + + Assert.NotEmpty(processed["rows"]!); + } + + // ---------------------------------------------------------------- validating + + [SkippableFact] + public async Task Valid_sql_passes_validation() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = $"select id from {MySqlDbFixture.Table} where id = @id" }, + timeoutSeconds: 60); + + Assert.True(result.Value("ok"), result.Value("error")); + } + + [SkippableFact] + public async Task A_dropped_table_is_caught_before_the_statement_is_stored() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "select 1 from nothing_of_the_sort" }, timeoutSeconds: 60); + + Assert.False(result.Value("ok")); + Assert.False(string.IsNullOrWhiteSpace(result.Value("error"))); + } + + /// + /// The mistake worth naming rather than leaving to a character offset: SQL copied from an + /// Oracle data source, where a parameter is :name, into one where it is @name. + /// + [SkippableFact] + public async Task The_wrong_placeholder_prefix_is_named() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = $"select id from {MySqlDbFixture.Table} where id = :ident" }, + timeoutSeconds: 60); + + Assert.False(result.Value("ok")); + Assert.Contains("@ident", result.Value("error")); + } + + [SkippableFact] + public async Task A_bare_procedure_name_is_accepted_and_says_what_was_not_checked() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "some_procedure" }, timeoutSeconds: 60); + + Assert.True(result.Value("ok")); + Assert.Contains("existence not checked", result.Value("note")); + } + + // ---------------------------------------------------------------- setup + + static int KeyOf(string fileId) => int.Parse(fileId.Substring(fileId.IndexOf(':') + 1)); + + static Dictionary Subscription(int id) => + new() { ["__subscriptionId__"] = id.ToString() }; + + async Task CursorAsync(string name) => + await _fixture.App.Services.GetRequiredService() + .GetAsync(new AdapterStateKey + { + AdapterId = BusAdapters.MySql, + InstanceKey = _dataSourceId.ToString(), + Name = name + }, default); + + async Task CreateDataSourceAsync() + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var dataSource = new DataSource + { + Name = $"mysql-{Guid.NewGuid():N}", + AdapterId = BusAdapters.MySql, + Kind = DataSourceKind.Relational, + Properties = Properties(), + SecretProperties = ["Password"] + }; + + db.Add(dataSource); + await db.SaveChangesAsync(); + return dataSource.Id; + } + + Dictionary Properties() => new() + { + ["Host"] = _mysql.Host, + ["Port"] = _mysql.Port.ToString(), + ["Database"] = _mysql.Database, + ["UserName"] = _mysql.User, + ["Password"] = _mysql.Password, + ["MinPoolSize"] = "1", + ["MaxPoolSize"] = "5", + + ["Statements"] = $@"{{ + ""seededOrders"": ""select * from {MySqlDbFixture.Table} where id <= 25 order by id"", + ""ordersForCustomer"": ""select * from {MySqlDbFixture.Table} where customer = @customer order by id"", + ""ordersByCustomerProc"": ""orders_by_customer"", + ""processedOrders"": ""select * from {MySqlDbFixture.Table} where processed = 1 order by id"", + ""insertOrder"": ""insert into {MySqlDbFixture.Table} (id, customer, amount) values (@id, @customer, @amount)"", + + ""earlyOrders"": {{ + ""sql"": ""select * from {MySqlDbFixture.Table} where id > @cursor and id <= 10 order by id"", + ""cursorColumn"": ""id"", + ""keyColumn"": ""id"" + }} + }}", + + ["ReceiveMode"] = "incrementing", + ["ReceiveStatement"] = + $"select * from {MySqlDbFixture.Table} where id > @cursor order by id limit 5", + ["CursorColumn"] = "id", + ["KeyColumn"] = "id", + ["MarkProcessedStatement"] = + $"update {MySqlDbFixture.Table} set processed = 1 where id = @key", + ["ReceiveBatchSize"] = "5" + }; + + async Task StartAdapterAsync() + { + var spec = new AdapterSpec + { + AdapterId = BusAdapters.MySql, + InstanceKey = _dataSourceId.ToString() + }; + + foreach (var kv in Properties()) spec.StartupValues[kv.Key] = kv.Value; + + await Host.StartExclusiveAsync(spec); + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/OracleAdapterTests.cs b/SW.Bitween.IntegrationTests/Tests/OracleAdapterTests.cs index 03ef1a0d..835c606a 100644 --- a/SW.Bitween.IntegrationTests/Tests/OracleAdapterTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/OracleAdapterTests.cs @@ -427,6 +427,69 @@ await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 120, Assert.Equal(new[] { 6, 7, 8, 9, 10 }, keys); } + // ---------------------------------------------------------------- validating + + /// + /// ODP.NET's Prepare is a client-side no-op — Oracle compiles a statement when it is executed, + /// not when it is prepared — so the shared check passed EVERYTHING here, including a select + /// from a table that does not exist. These pin the replacement: DBMS_SQL.PARSE, which compiles + /// and resolves names without running anything. + /// + [SkippableFact] + public async Task Valid_sql_passes_validation() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = $"select id from {OracleFixture.Table} where id = :id" }, + timeoutSeconds: 120); + + Assert.True(result.Value("ok"), result.Value("error")); + } + + [SkippableFact] + public async Task A_dropped_table_is_caught_before_the_statement_is_stored() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "select 1 from nothing_of_the_sort" }, timeoutSeconds: 120); + + Assert.False(result.Value("ok")); + + // ORA-00942: table or view does not exist. Named rather than matched on words, because the + // message is localised and the number is not. + Assert.Contains("ORA-00942", result.Value("error")); + } + + [SkippableFact] + public async Task A_syntax_error_is_caught() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = $"select id from {OracleFixture.Table} where id = @id" }, + timeoutSeconds: 120); + + Assert.False(result.Value("ok")); + + // @ is a database link on Oracle, not a placeholder, so this is ORA-00936 missing + // expression — and the hint says to write :id instead. + Assert.Contains(":id", result.Value("error")); + } + + [SkippableFact] + public async Task A_bare_procedure_name_is_accepted_and_says_what_was_not_checked() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "some_procedure" }, timeoutSeconds: 120); + + Assert.True(result.Value("ok")); + Assert.Contains("existence not checked", result.Value("note")); + } + // ---------------------------------------------------------------- setup async Task CreateDataSourceAsync() diff --git a/SW.Bitween.IntegrationTests/Tests/SqlServerAdapterTests.cs b/SW.Bitween.IntegrationTests/Tests/SqlServerAdapterTests.cs new file mode 100644 index 00000000..62c68fe4 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/SqlServerAdapterTests.cs @@ -0,0 +1,565 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Serverless.Resident; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The SQL Server data source provider, against a real SQL Server. +/// +/// The same suite as the others, which is the point: four engines run on one core, so the parts +/// that are shared should behave identically and the parts that differ should differ VISIBLY. +/// SQL Server is the one where the shared statement CHECK could not be shared — its driver refuses +/// to prepare a command whose parameters have no explicit type — so the check has its own tests +/// here rather than relying on the core's. +/// +[Collection("Bitween")] +public class SqlServerAdapterTests : IClassFixture +{ + readonly BitweenFixture _fixture; + readonly SqlServerDbFixture _sqlServer; + + public SqlServerAdapterTests(BitweenFixture fixture, SqlServerDbFixture sqlServer) + { + _fixture = fixture; + _sqlServer = sqlServer; + } + + static readonly SemaphoreSlim Gate = new(1, 1); + static int _dataSourceId; + + IResidentAdapterHost Host => _fixture.App.Services.GetRequiredService(); + + async Task AdapterAsync() + { + Skip.If(_sqlServer.Unavailable != null, $"SQL Server is not available: {_sqlServer.Unavailable}"); + + await Gate.WaitAsync(); + try + { + if (_dataSourceId == 0) + { + _dataSourceId = await CreateDataSourceAsync(); + await StartAdapterAsync(); + } + } + finally + { + Gate.Release(); + } + + return Host.Get(BusAdapters.SqlServer, _dataSourceId.ToString()) + ?? throw new InvalidOperationException("The SQL Server adapter is not running."); + } + + // ---------------------------------------------------------------- configuration + + [SkippableFact] + public async Task Connection_test_reports_each_stage_and_checks_every_statement() + { + var adapter = await AdapterAsync(); + var result = await adapter.InvokeAsync("TestConnection", timeoutSeconds: 60); + + Assert.True(result.Value("ok"), result.ToString()); + + var steps = result["steps"]!.Select(s => s.Value("step")).ToList(); + Assert.Contains("connect", steps); + Assert.Contains("authenticate", steps); + Assert.Contains("privileges", steps); + Assert.Contains("statement:ordersForCustomer", steps); + Assert.All(result["steps"]!, s => Assert.True(s.Value("ok"), s.ToString())); + } + + /// + /// SQL Server is the most capable of the four on paper, and the list says so where it is true — + /// MERGE, OUTPUT and snapshot isolation are all real here — and stays false where the thing + /// exists but is not wired, which is Service Broker's query notifications. + /// + [SkippableFact] + public async Task Describe_reports_the_engine_and_where_it_differs() + { + var adapter = await AdapterAsync(); + var described = await adapter.InvokeAsync("Describe", timeoutSeconds: 60); + + Assert.Equal("SQL Server", described.Value("engine")); + Assert.False(string.IsNullOrWhiteSpace(described.Value("serverVersion"))); + + Assert.True(described.Value("storedProcedures")); + Assert.True(described.Value("procedureResultSets")); + Assert.True(described.Value("merge")); + Assert.True(described.Value("returning")); + + // Table-valued parameters are not an array type: a caller cannot bind a list to one + // parameter the way PostgreSQL allows. + Assert.False(described.Value("arrayTypes")); + + var objects = described["supportedObjects"]!.Select(o => o.Value()).ToList(); + Assert.Contains("sequence", objects); + + var isolation = described["isolationLevels"]!.Select(i => i.Value()).ToList(); + Assert.Contains("Snapshot", isolation); + + // Exists, not wired. Declared false so the UI says "not available" rather than leaving a gap. + Assert.False(described.Value("changeNotification")); + Assert.False(described.Value("logBasedCdc")); + + Assert.NotEmpty(described["privileges"]!); + } + + // ---------------------------------------------------------------- discovery + + [SkippableFact] + public async Task Discover_finds_the_table_with_its_columns_and_key() + { + var adapter = await AdapterAsync(); + var result = await adapter.InvokeAsync("Discover", new + { + objectType = "table", + schema = "sales", + nameLike = SqlServerDbFixture.Table, + includeColumns = true, + includeRowCounts = true + }, timeoutSeconds: 60); + + var table = result["objects"]!.Single(o => o.Value("name") == SqlServerDbFixture.Table); + + Assert.Equal("sales", table.Value("schema")); + + // An extended property, which is where SQL Server keeps what other engines call a comment. + Assert.Equal("Customer orders", table.Value("comment")); + + // The partition stats, not COUNT(*) — so this is an estimate, and asserting an exact + // number would be asserting something the adapter deliberately does not promise. Other + // tests in this class insert rows too, which is the second reason: 25 is the floor. + Assert.True(table.Value("rowCount") >= 25, + $"expected at least the 25 seeded rows, got {table.Value("rowCount")}"); + + var columns = table["columns"]!.ToList(); + Assert.Equal("id", columns[0].Value("name")); + Assert.True(columns[0].Value("primaryKey")); + Assert.False(columns[0].Value("nullable")); + + // max_length is in BYTES and an nvarchar stores two per character, so the declared length + // is half of it — 50, not 100. + var customer = columns.Single(c => c.Value("name") == "customer"); + Assert.Equal("nvarchar(50)", customer.Value("dbType")); + Assert.Equal(50, customer.Value("length")); + + var amount = columns.Single(c => c.Value("name") == "amount"); + Assert.Equal("decimal(10,2)", amount.Value("dbType")); + Assert.Equal("decimal", amount.Value("clrType")); + + // A default constraint is a value the database fills in. + var created = columns.Single(c => c.Value("name") == "created_at"); + Assert.True(created.Value("generated")); + + var processed = columns.Single(c => c.Value("name") == "processed"); + Assert.Equal("bool", processed.Value("clrType")); + } + + /// + /// The schema filter does something, which is why the fixture puts everything in `sales` rather + /// than dbo: against a schema-less seed a filter that was ignored would still pass. + /// + [SkippableFact] + public async Task Discover_filters_by_schema() + { + var adapter = await AdapterAsync(); + + var inSales = await adapter.InvokeAsync("Discover", + new { objectType = "table", schema = "sales" }, timeoutSeconds: 60); + Assert.NotEmpty(inSales["objects"]!); + + var inDbo = await adapter.InvokeAsync("Discover", + new { objectType = "table", schema = "dbo" }, timeoutSeconds: 60); + Assert.Empty(inDbo["objects"]!); + } + + [SkippableFact] + public async Task Discover_finds_procedures_functions_and_sequences() + { + var adapter = await AdapterAsync(); + + var procedures = await adapter.InvokeAsync("Discover", + new { objectType = "procedure", schema = "sales" }, timeoutSeconds: 60); + + var counting = procedures["objects"]!.Single(o => o.Value("name") == "count_orders"); + var parameters = counting["parameters"]!.ToList(); + + // The @ is stripped: a caller binds by name, and the prefix is the driver's business. + Assert.Equal("p_customer", parameters[0].Value("name")); + Assert.Equal("In", parameters[0].Value("direction")); + Assert.Equal("p_total", parameters[1].Value("name")); + Assert.Equal("Out", parameters[1].Value("direction")); + + // An inline table-valued function is a function, not a procedure — the shape a receive + // statement would select from. + var functions = await adapter.InvokeAsync("Discover", + new { objectType = "function", schema = "sales" }, timeoutSeconds: 60); + Assert.Contains(functions["objects"]!, o => o.Value("name") == "orders_for"); + + var sequences = await adapter.InvokeAsync("Discover", + new { objectType = "sequence", schema = "sales" }, timeoutSeconds: 60); + Assert.Contains(sequences["objects"]!, o => o.Value("name") == "bitween_order_seq"); + } + + // ---------------------------------------------------------------- running + + [SkippableFact] + public async Task Query_returns_rows_for_a_named_statement() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Query", new + { + name = "ordersForCustomer", + parameters = new Dictionary { ["customer"] = "acme" } + }, timeoutSeconds: 60); + + var rows = result["rows"]!.ToList(); + Assert.NotEmpty(rows); + Assert.All(rows, r => Assert.Equal("acme", r.Value("customer"))); + } + + [SkippableFact] + public async Task Call_returns_rows_straight_from_a_procedure() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Call", new + { + name = "ordersByCustomerProc", + parameters = new Dictionary { ["p_customer"] = "globex" } + }, timeoutSeconds: 60); + + var rows = result["rows"]!.ToList(); + Assert.NotEmpty(rows); + Assert.All(rows, r => Assert.Equal("globex", r.Value("customer"))); + } + + /// + /// OUTPUT is SQL Server's RETURNING, and the capability list claims it — so a statement using + /// it has to actually work, not merely be declared possible. + /// + [SkippableFact] + public async Task An_insert_with_output_returns_the_row_it_wrote() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Query", new + { + name = "insertOrderOutput", + parameters = new Dictionary + { ["id"] = 902, ["customer"] = "outputted", ["amount"] = 5.5 } + }, timeoutSeconds: 60); + + var rows = result["rows"]!.ToList(); + Assert.Single(rows); + Assert.Equal("outputted", rows[0].Value("customer")); + } + + [SkippableFact] + public async Task Execute_reports_what_it_changed() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("Execute", new + { + name = "insertOrder", + parameters = new Dictionary + { ["id"] = 801, ["customer"] = "inserted", ["amount"] = 12.34 } + }, timeoutSeconds: 60); + + Assert.Equal(1, result.Value("affectedRows")); + } + + [SkippableFact] + public async Task A_failed_batch_rolls_back_what_came_before_it() + { + var adapter = await AdapterAsync(); + + await Assert.ThrowsAnyAsync(() => adapter.InvokeAsync("Batch", new + { + statements = new object[] + { + new + { + name = "insertOrder", + parameters = new Dictionary + { ["id"] = 850, ["customer"] = "rolled-back", ["amount"] = 1 } + }, + new + { + // id 1 is seeded, so this violates the primary key. + name = "insertOrder", + parameters = new Dictionary + { ["id"] = 1, ["customer"] = "rolled-back", ["amount"] = 1 } + } + } + }, timeoutSeconds: 60)); + + var back = await adapter.InvokeAsync("Query", new + { + name = "ordersForCustomer", + parameters = new Dictionary { ["customer"] = "rolled-back" } + }, timeoutSeconds: 60); + + Assert.Empty(back["rows"]!); + } + + [SkippableFact] + public async Task Sql_sent_with_a_message_is_refused() + { + var adapter = await AdapterAsync(); + + var error = await Assert.ThrowsAnyAsync(() => + adapter.InvokeAsync("Query", + new { sql = $"select * from sales.{SqlServerDbFixture.Table}" }, timeoutSeconds: 60)); + + Assert.Contains("does not allow ad-hoc SQL", error.Message); + } + + // ---------------------------------------------------------------- receiving + + [SkippableFact] + public async Task The_receiver_advances_a_cursor_that_survives_a_restart() + { + var adapter = await AdapterAsync(); + var me = Subscription(1); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: me); + var first = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: me); + + Assert.Equal(5, first.Count); + + foreach (var id in first) + { + var file = await adapter.InvokeAsync("GetFile", id, timeoutSeconds: 60, + properties: me); + var data = file.Value("data") ?? file.Value("Data"); + Assert.False(string.IsNullOrWhiteSpace(data)); + + await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 60, properties: me); + } + + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: me); + + Assert.Equal("5", await CursorAsync("receive.cursor.1")); + + var restarted = await Host.RestartAsync(BusAdapters.SqlServer, _dataSourceId.ToString(), + drain: false); + + await restarted.InvokeAsync("Initialize", timeoutSeconds: 60, properties: me); + var second = await restarted.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: me); + + Assert.Equal(new[] { 6, 7, 8, 9, 10 }, second.Select(KeyOf).OrderBy(i => i)); + } + + [SkippableFact] + public async Task Two_subscriptions_on_one_data_source_do_not_share_a_cursor() + { + var adapter = await AdapterAsync(); + + var first = Subscription(101); + var second = Subscription(202); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: first); + var forFirst = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: first); + foreach (var id in forFirst) + await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 60, properties: first); + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: first); + + Assert.NotEmpty(forFirst); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: second); + var forSecond = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: second); + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: second); + + Assert.Equal( + forFirst.Select(KeyOf).OrderBy(k => k), + forSecond.Select(KeyOf).OrderBy(k => k)); + } + + [SkippableFact] + public async Task Mark_processed_runs_for_each_accepted_row() + { + var adapter = await AdapterAsync(); + var me = Subscription(2); + + await adapter.InvokeAsync("Initialize", timeoutSeconds: 60, properties: me); + var listed = await adapter.InvokeAsync>("ListFiles", timeoutSeconds: 60, + properties: me); + + foreach (var id in listed) + await adapter.InvokeAsync("DeleteFile", id, timeoutSeconds: 60, properties: me); + + await adapter.InvokeAsync("Finalize", timeoutSeconds: 60, properties: me); + + var processed = await adapter.InvokeAsync("Query", new { name = "processedOrders" }, + timeoutSeconds: 60); + + Assert.NotEmpty(processed["rows"]!); + } + + // ---------------------------------------------------------------- validating + + /// + /// The check that could not be shared. SqlCommand.Prepare refuses unless every parameter has an + /// explicit type, which a caller checking someone else's SQL does not know — so this adapter + /// asks sp_describe_undeclared_parameters instead, and these prove it answers the same + /// questions the prepared form does everywhere else. + /// + [SkippableFact] + public async Task Valid_sql_passes_validation_even_with_parameters() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = $"select id from sales.{SqlServerDbFixture.Table} where id = @id" }, + timeoutSeconds: 60); + + Assert.True(result.Value("ok"), result.Value("error")); + } + + [SkippableFact] + public async Task A_dropped_table_is_caught_before_the_statement_is_stored() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "select 1 from sales.nothing_of_the_sort" }, timeoutSeconds: 60); + + Assert.False(result.Value("ok")); + Assert.Contains("Invalid object name", result.Value("error")); + } + + [SkippableFact] + public async Task A_syntax_error_is_caught() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "selct 1" }, timeoutSeconds: 60); + + Assert.False(result.Value("ok")); + Assert.False(string.IsNullOrWhiteSpace(result.Value("error"))); + } + + [SkippableFact] + public async Task A_bare_procedure_name_is_accepted_and_says_what_was_not_checked() + { + var adapter = await AdapterAsync(); + + var result = await adapter.InvokeAsync("ValidateStatement", + new { sql = "sales.some_procedure" }, timeoutSeconds: 60); + + Assert.True(result.Value("ok")); + Assert.Contains("existence not checked", result.Value("note")); + } + + // ---------------------------------------------------------------- setup + + static int KeyOf(string fileId) => int.Parse(fileId.Substring(fileId.IndexOf(':') + 1)); + + static Dictionary Subscription(int id) => + new() { ["__subscriptionId__"] = id.ToString() }; + + async Task CursorAsync(string name) => + await _fixture.App.Services.GetRequiredService() + .GetAsync(new AdapterStateKey + { + AdapterId = BusAdapters.SqlServer, + InstanceKey = _dataSourceId.ToString(), + Name = name + }, default); + + async Task CreateDataSourceAsync() + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var dataSource = new DataSource + { + Name = $"sqlserver-{Guid.NewGuid():N}", + AdapterId = BusAdapters.SqlServer, + Kind = DataSourceKind.Relational, + Properties = Properties(), + SecretProperties = ["Password"] + }; + + db.Add(dataSource); + await db.SaveChangesAsync(); + return dataSource.Id; + } + + Dictionary Properties() => new() + { + ["Host"] = _sqlServer.Host, + ["Port"] = _sqlServer.Port.ToString(), + ["Database"] = _sqlServer.Database, + ["UserName"] = _sqlServer.User, + ["Password"] = _sqlServer.Password, + ["Schema"] = "sales", + ["MinPoolSize"] = "1", + ["MaxPoolSize"] = "5", + + // The container presents a self-signed certificate, which is the case this setting exists + // for — and the reason the adapter offers it rather than pretending every server has a + // certificate chain that validates. + ["Encrypt"] = "true", + ["TrustServerCertificate"] = "true", + + ["Statements"] = $@"{{ + ""seededOrders"": ""select * from sales.{SqlServerDbFixture.Table} where id <= 25 order by id"", + ""ordersForCustomer"": ""select * from sales.{SqlServerDbFixture.Table} where customer = @customer order by id"", + ""ordersByCustomerProc"": ""sales.orders_by_customer"", + ""ordersForFunction"": ""select * from sales.orders_for(@customer)"", + ""processedOrders"": ""select * from sales.{SqlServerDbFixture.Table} where processed = 1 order by id"", + ""insertOrder"": ""insert into sales.{SqlServerDbFixture.Table} (id, customer, amount) values (@id, @customer, @amount)"", + ""insertOrderOutput"": ""insert into sales.{SqlServerDbFixture.Table} (id, customer, amount) output inserted.* values (@id, @customer, @amount)"", + + ""earlyOrders"": {{ + ""sql"": ""select * from sales.{SqlServerDbFixture.Table} where id > @cursor and id <= 10 order by id"", + ""cursorColumn"": ""id"", + ""keyColumn"": ""id"" + }} + }}", + + ["ReceiveMode"] = "incrementing", + + // TOP rather than LIMIT, and it has to come before the column list — one of the small + // dialect differences a statement writer meets immediately. + ["ReceiveStatement"] = + $"select top 5 * from sales.{SqlServerDbFixture.Table} where id > @cursor order by id", + ["CursorColumn"] = "id", + ["KeyColumn"] = "id", + ["MarkProcessedStatement"] = + $"update sales.{SqlServerDbFixture.Table} set processed = 1 where id = @key", + ["ReceiveBatchSize"] = "5" + }; + + async Task StartAdapterAsync() + { + var spec = new AdapterSpec + { + AdapterId = BusAdapters.SqlServer, + InstanceKey = _dataSourceId.ToString() + }; + + foreach (var kv in Properties()) spec.StartupValues[kv.Key] = kv.Value; + + await Host.StartExclusiveAsync(spec); + } +} diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index bfff67f1..9daef0e2 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -72,6 +72,7 @@ import type { TrailEntry, WorkGroupRow, } from "./types"; +import type { SaveResult } from "./http/dataSourceStatements"; /** * The single data-access contract the UI is written against, implemented by @@ -378,7 +379,7 @@ export interface ApiClient { cursorColumn?: string | null; keyColumn?: string | null; }, - ): Promise<{ id: number }>; + ): Promise; updateDataSourceStatement( id: number, changes: { @@ -390,7 +391,7 @@ export interface ApiClient { cursorColumn?: string | null; keyColumn?: string | null; }, - ): Promise; + ): Promise; deleteDataSourceStatement(id: number): Promise; getDataSourceStatementUsage(id: number): Promise; /** The subscription is either an existing id or defined inline; the endpoint commits both as one. */ diff --git a/SW.Bitween.Web/ClientApp/src/api/http/dataSourceStatements.ts b/SW.Bitween.Web/ClientApp/src/api/http/dataSourceStatements.ts index 568e93ca..7cf52b0c 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/dataSourceStatements.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/dataSourceStatements.ts @@ -8,6 +8,17 @@ interface SearchyResponse { totalCount: number; } +/** + * What a save answers with. `checked` is false when the database was never asked — the adapter for + * this connection is not running on the node that handled the request, so there was nobody to + * validate against. Worth saying: a save that reported nothing would look exactly like one that + * had been verified. + */ +export interface SaveResult { + id: number; + checked: boolean; +} + interface RawStatement { id: number; dataSourceId: number; @@ -70,10 +81,10 @@ export const dataSourceStatementMethods: Partial = { cursorColumn?: string | null; keyColumn?: string | null; }, - ): Promise<{ id: number }> { + ): Promise { // The data source travels in the body, not the route: POST /datasourcestatements/{id} already // means "update that statement", so a keyed create would collide with it. - const id = await post(`/datasourcestatements`, { + const saved = await post(`/datasourcestatements`, { dataSourceId, name: input.name, sql: input.sql, @@ -83,7 +94,7 @@ export const dataSourceStatementMethods: Partial = { keyColumn: input.keyColumn || null, inactive: false, }); - return { id }; + return saved; }, async updateDataSourceStatement( @@ -97,8 +108,8 @@ export const dataSourceStatementMethods: Partial = { cursorColumn?: string | null; keyColumn?: string | null; }, - ): Promise { - await post(`/datasourcestatements/${id}`, { + ): Promise { + return post(`/datasourcestatements/${id}`, { name: changes.name, sql: changes.sql, description: changes.description ?? null, diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/SchemaBrowser.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/SchemaBrowser.tsx index 0930c34c..c63bb098 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/data-sources/SchemaBrowser.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/SchemaBrowser.tsx @@ -6,6 +6,7 @@ import { Badge, Button, FormError, LoadingBlock } from "../../components/ui/basi import { Select, TextInput } from "../../components/ui/forms"; import { Panel } from "../../components/ui/Panel"; import { + dialectOf, draftNameFor, draftStatementFor, fetchCapabilities, @@ -13,6 +14,7 @@ import { fetchSchemaPage, groupBySchema, type DbObject, + type SqlDialect, } from "./schema"; /** How many objects one page asks for. The adapter clamps anything above 1000. */ @@ -92,6 +94,10 @@ export function SchemaBrowser({ const rows = objects.data?.objects ?? []; const hasMore = objects.data?.hasMore ?? false; + // How to write a placeholder and a row limit for THIS engine. Reported by the adapter rather + // than guessed from the label, so a draft cannot use a prefix the connection will refuse. + const dialect = dialectOf(capabilities.data); + return ( ))} @@ -216,6 +223,7 @@ const LABELS: Record = { function: "Functions", sequence: "Sequences", package: "Packages", + synonym: "Synonyms", }; /** @@ -225,10 +233,12 @@ const LABELS: Record = { function ObjectRow({ dataSourceId, object, + dialect, onUseInStatement, }: { dataSourceId: number; object: DbObject; + dialect: SqlDialect; onUseInStatement?: (draft: { name: string; sql: string; description: string }) => void; }) { const [open, setOpen] = useState(false); @@ -283,7 +293,7 @@ function ObjectRow({ onClick={() => onUseInStatement({ name: draftNameFor(full), - sql: draftStatementFor(full), + sql: draftStatementFor(full, dialect), description: `Generated from ${full.schema}.${full.name}.`, }) } diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/Statements.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/Statements.tsx index 9315d057..231a8ad1 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/data-sources/Statements.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/Statements.tsx @@ -306,10 +306,16 @@ function StatementForm({ // looks like a button that did nothing. const [justSaved, setJustSaved] = useState(false); + // True when the database never looked at the SQL — the adapter for this connection is not + // running, so there was nobody to ask. Said out loud because a save that reported nothing would + // look exactly like one that had been verified, and the next thing to notice would be a failed + // connection test or a failed message. + const [unchecked, setUnchecked] = useState(false); + const save = useMutation({ mutationFn: async () => { if (statement) { - await api.updateDataSourceStatement(statement.id, { + return api.updateDataSourceStatement(statement.id, { name, sql, description, @@ -318,18 +324,19 @@ function StatementForm({ cursorColumn, keyColumn, }); - } else { - await api.createDataSourceStatement(dataSourceId, { - name, - sql, - description, - cursorColumn, - keyColumn, - }); } + + return api.createDataSourceStatement(dataSourceId, { + name, + sql, + description, + cursorColumn, + keyColumn, + }); }, - onSuccess: () => { + onSuccess: (saved) => { setError(null); + setUnchecked(!saved.checked); // Only for an edit. A create closes the form on success, which says it landed by itself — // and a "Saved" flash on a form that is disappearing is a flicker, not a message. if (statement) { @@ -436,6 +443,13 @@ function StatementForm({ where the eye already is when the save is pressed. */} {error && {error}} + {unchecked && !error && ( +

+ Saved, but not checked. This connection is not running here, so the database never saw + this SQL — run a connection test once it is up. +

+ )} +
{justSaved ? ( diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/__tests__/schema.test.ts b/SW.Bitween.Web/ClientApp/src/pages/data-sources/__tests__/schema.test.ts index 47d9e281..f9bdfd27 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/data-sources/__tests__/schema.test.ts +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/__tests__/schema.test.ts @@ -65,7 +65,7 @@ describe("draftStatementFor", () => { expect(draftStatementFor(object())).toContain("select *"); }); - it("calls a procedure rather than selecting from it", () => { + it("names a procedure rather than writing a CALL for it", () => { const sql = draftStatementFor( object({ type: "procedure", @@ -77,10 +77,47 @@ describe("draftStatementFor", () => { }), ); - // Only the inputs are bound: an out parameter is the procedure's answer, not something the - // caller supplies. The adapter sends the direction as `In`/`Out`, so the match is - // case-insensitive — reading it literally would bind everything or nothing. - expect(sql).toBe("call sales.release_order(@order_id)"); + // A statement meant for Call holds the procedure's NAME, not SQL — that is what + // CommandType.StoredProcedure takes, and on Oracle it is the only form that works. Drafting + // "call sales.release_order(@order_id)" made a statement whose procedure name was that entire + // string, which fails on first use and nowhere near here. + expect(sql).toBe("sales.release_order"); + }); + + it("writes the placeholder and the row limit the engine actually takes", () => { + const table = object({ + columns: [ + { name: "id", dbType: "integer", clrType: "Int32", nullable: false, primaryKey: true, generated: true, ordinal: 1 }, + ], + }); + + // Oracle: fetch first, and : for a bind. + expect(draftStatementFor(table, { parameterPrefix: ":", limitStyle: "fetchFirst" })) + .toContain("fetch first 100 rows only"); + + // SQL Server: TOP, and it goes BEFORE the columns rather than after the query. + expect(draftStatementFor(table, { parameterPrefix: "@", limitStyle: "top" })) + .toContain("select top 100 id"); + + const fn = object({ + type: "function", + name: "orders_for", + parameters: [{ name: "code", dbType: "varchar2", direction: "In", ordinal: 1 }], + }); + + expect(draftStatementFor(fn, { parameterPrefix: ":", limitStyle: "fetchFirst" })) + .toBe("select * from sales.orders_for(:code)"); + }); + + it("reads the next value of a sequence the way each engine spells it", () => { + const seq = object({ type: "sequence", name: "order_seq" }); + + expect(draftStatementFor(seq, { parameterPrefix: "@", limitStyle: "limit" })) + .toBe("select nextval('sales.order_seq')"); + expect(draftStatementFor(seq, { parameterPrefix: "@", limitStyle: "top" })) + .toBe("select next value for sales.order_seq"); + expect(draftStatementFor(seq, { parameterPrefix: ":", limitStyle: "fetchFirst" })) + .toBe("select sales.order_seq.nextval from dual"); }); it("selects from a set-returning function", () => { @@ -95,11 +132,7 @@ describe("draftStatementFor", () => { expect(sql).toBe("select * from sales.orders_for_customer(@cid)"); }); - it("reads the next value of a sequence", () => { - expect(draftStatementFor(object({ type: "sequence", name: "order_seq" }))).toBe( - "select nextval('sales.order_seq')", - ); - }); + }); describe("draftNameFor", () => { diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/schema.ts b/SW.Bitween.Web/ClientApp/src/pages/data-sources/schema.ts index c115fb2b..fe6642f8 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/data-sources/schema.ts +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/schema.ts @@ -61,8 +61,33 @@ export interface DbCapabilities { supportedObjects: string[]; schemaDiscovery: boolean; rowCountEstimates: boolean; + /** `:` or `@`. The adapter reports its own, so a draft cannot use the wrong one. */ + parameterPrefix: string; + /** How this engine limits rows: `limit`, `fetchFirst` or `top`. They are not interchangeable. */ + limitStyle: string; } +/** + * Everything a generated statement needs to know about the engine it is for. + * + * The browser used to write `@name` and `limit 100` whatever the connection was, which is right + * for PostgreSQL and MySQL and wrong for the other two: Oracle binds `:name` and takes + * `fetch first N rows only`, and SQL Server puts `top N` before the column list. A draft that + * does not parse is worse than no draft, because it looks like something that was checked. + */ +export interface SqlDialect { + parameterPrefix: string; + limitStyle: string; +} + +/** PostgreSQL's, and the safe assumption while the capability list is still loading. */ +export const DEFAULT_DIALECT: SqlDialect = { parameterPrefix: "@", limitStyle: "limit" }; + +export const dialectOf = (capabilities: DbCapabilities | undefined): SqlDialect => ({ + parameterPrefix: capabilities?.parameterPrefix || DEFAULT_DIALECT.parameterPrefix, + limitStyle: capabilities?.limitStyle || DEFAULT_DIALECT.limitStyle, +}); + /** * Thrown when the adapter answered but not with what was expected. Separate from a transport * failure because the remedy is different: this one is a bug or a version mismatch, not a @@ -198,27 +223,46 @@ export const qualify = (schema: string, name: string): string => * thing the catalog cannot tell us, and a row limit, because the first thing anyone does with a * new statement is run it against a table whose size they do not know. */ -export const draftStatementFor = (object: DbObject): string => { +export const draftStatementFor = ( + object: DbObject, + dialect: SqlDialect = DEFAULT_DIALECT, +): string => { const target = qualify(object.schema, object.name); + const bind = (name: string) => `${dialect.parameterPrefix}${name}`; if (object.type === "procedure" || object.type === "function") { // Only what the caller supplies. An out parameter, a return value and a REF CURSOR are the // routine's answer — binding them as inputs is how a generated call fails on first run. const args = object.parameters .filter((p) => SUPPLIED.has(p.direction?.toLowerCase())) - .map((p) => `@${p.name}`) + .map((p) => bind(p.name)) .join(", "); + + // A procedure is named, not written as SQL: that is what CommandType.StoredProcedure takes, + // and on Oracle it is the only form that works. The parameters are bound by the message. return object.type === "function" ? `select * from ${target}(${args})` - : `call ${target}(${args})`; + : target; } - if (object.type === "sequence") return `select nextval('${object.schema}.${object.name}')`; + if (object.type === "sequence") + return dialect.limitStyle === "limit" + ? `select nextval('${object.schema}.${object.name}')` + : dialect.limitStyle === "top" + ? `select next value for ${target}` + : `select ${target}.nextval from dual`; const columns = object.columns.length ? [...object.columns].sort((a, b) => a.ordinal - b.ordinal).map((c) => c.name).join(", ") : "*"; - return `select ${columns}\n from ${target}\n limit 100`; + + // Three shapes, and they are not interchangeable — TOP goes before the columns, the other two + // after the query. + if (dialect.limitStyle === "top") + return `select top 100 ${columns}\n from ${target}`; + + const tail = dialect.limitStyle === "fetchFirst" ? "fetch first 100 rows only" : "limit 100"; + return `select ${columns}\n from ${target}\n ${tail}`; }; /** A name for the statement, derived from the object so two objects never collide. */ diff --git a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/DataSourceBinding.tsx b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/DataSourceBinding.tsx index a02c1b20..57b6517c 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/DataSourceBinding.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/DataSourceBinding.tsx @@ -333,6 +333,10 @@ function StatementColumns({ setError(null); }, [statement.id, statement.keyColumn, statement.cursorColumn]); + // True when the database never looked at the SQL — see the statements panel, which says the + // same thing for the same reason. + const [unchecked, setUnchecked] = useState(false); + const save = useMutation({ mutationFn: () => api.updateDataSourceStatement(statement.id, { @@ -344,8 +348,9 @@ function StatementColumns({ keyColumn, cursorColumn, }), - onSuccess: () => { + onSuccess: (saved) => { setError(null); + setUnchecked(!saved.checked); setJustSaved(true); setTimeout(() => setJustSaved(false), 2000); void queryClient.invalidateQueries({ @@ -396,6 +401,13 @@ function StatementColumns({ {error && {error}} + {unchecked && !error && ( +

+ Saved, but not checked — this connection is not running here, so the database never saw + the change. +

+ )} + {!canEdit && (

Changing these needs the right to edit this connection's statements. diff --git a/SW.Bitween.sln b/SW.Bitween.sln index 54e43a51..3111b237 100644 --- a/SW.Bitween.sln +++ b/SW.Bitween.sln @@ -45,6 +45,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Db.Orac EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Db.PostgreSql", "SW.Bitween.Adapters.Db.PostgreSql\SW.Bitween.Adapters.Db.PostgreSql.csproj", "{E84711FC-C53F-4B03-88DC-598D96F8D756}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Db.MySql", "SW.Bitween.Adapters.Db.MySql\SW.Bitween.Adapters.Db.MySql.csproj", "{6223A726-85F6-4280-9B45-854633591E64}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Db.SqlServer", "SW.Bitween.Adapters.Db.SqlServer\SW.Bitween.Adapters.Db.SqlServer.csproj", "{F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -271,6 +275,30 @@ Global {E84711FC-C53F-4B03-88DC-598D96F8D756}.Release|x64.Build.0 = Release|Any CPU {E84711FC-C53F-4B03-88DC-598D96F8D756}.Release|x86.ActiveCfg = Release|Any CPU {E84711FC-C53F-4B03-88DC-598D96F8D756}.Release|x86.Build.0 = Release|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Debug|x64.ActiveCfg = Debug|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Debug|x64.Build.0 = Debug|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Debug|x86.ActiveCfg = Debug|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Debug|x86.Build.0 = Debug|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Release|Any CPU.Build.0 = Release|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Release|x64.ActiveCfg = Release|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Release|x64.Build.0 = Release|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Release|x86.ActiveCfg = Release|Any CPU + {6223A726-85F6-4280-9B45-854633591E64}.Release|x86.Build.0 = Release|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Debug|x64.ActiveCfg = Debug|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Debug|x64.Build.0 = Debug|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Debug|x86.ActiveCfg = Debug|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Debug|x86.Build.0 = Debug|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Release|Any CPU.Build.0 = Release|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Release|x64.ActiveCfg = Release|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Release|x64.Build.0 = Release|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Release|x86.ActiveCfg = Release|Any CPU + {F5EA521E-1B9C-4723-8747-D4A6BB9D4EDA}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docs/database-adapters-api.md b/docs/database-adapters-api.md index 855763fd..c864c710 100644 --- a/docs/database-adapters-api.md +++ b/docs/database-adapters-api.md @@ -58,6 +58,49 @@ Settings marked **secret** are encrypted at rest and never returned by the API. | `ReceiveBatchSize` | 500 | Default rows one poll takes; a subscription may override. | | `ReceiveStatement`, `CursorColumn`, `KeyColumn`, `MarkProcessedStatement` | — | **Legacy.** Still honoured if set, hidden from the form. See §6.1 for where each now lives. | +### 2.1a MySQL — `bitween.db.mysql` + +Serves MariaDB too. `Database` is also the schema — MySQL has no separate one — so it is what an +unqualified name resolves against, and a discovery call with no schema means the database this +connection opened rather than every database on the server. + +| Setting | Default | Notes | +|---|---|---| +| `Host`, `Port` | localhost, 3306 | | +| `Database` | — | Required. Also the schema. | +| `UserName`, `Password` | — | | +| `SslMode` | Preferred | Required or above off localhost. | +| `ConnectionIdleLifetimeSeconds` | 180 | Keep below the server's `wait_timeout`; a connection the server closed first surfaces as a broken pipe on the next message. | +| `ServerPrepare` | true | Off for a connection through ProxySQL, where prepared statements pin a backend. Also what makes the save-time check real — with it off, `Prepare` does nothing and every statement reports valid. | +| `AllowZeroDateTime` | false | MySQL permits `0000-00-00`, which no .NET date type holds. | + +No sequences (AUTO_INCREMENT belongs to a column), no MERGE, no RETURNING, no array type. A +procedure returns result sets by SELECTing — the capability PostgreSQL declares false. + +### 2.1b SQL Server — `bitween.db.sqlserver` + +Serves Azure SQL too. The default schema belongs to the LOGIN rather than the connection, so +`Schema` is applied per connection with `EXECUTE AS` — and failure is not fatal, because +impersonating a schema's owner is a privilege many service accounts do not have. + +| Setting | Default | Notes | +|---|---|---| +| `Host`, `Port` | localhost, 1433 | `Port` is ignored for `host\instance`, which resolves through the Browser service. | +| `Database` | — | Required. | +| `UserName`, `Password` | — | | +| `Schema` | — | Run as this schema, for unqualified names. | +| `Encrypt` | true | On by default in the modern driver — a change from the old one, and why an instance that worked for years fails on upgrade. | +| `TrustServerCertificate` | false | Needed for the usual self-signed on-premises certificate. Use knowingly. | +| `MultipleActiveResultSets` | false | Stops the connection resetting cleanly between uses, which is the opposite of what a shared pool wants. | +| `SnapshotIsolation` | false | Readers do not block writers. Needs the database to have it enabled. | + +MERGE, OUTPUT and snapshot isolation are all real here. Query Notifications over Service Broker +exists and is not wired, so `changeNotification` is false. + +**Its statement check is not the shared one.** `SqlCommand.Prepare` refuses unless every parameter +has an explicit type — which a caller checking somebody else's SQL does not know — so this adapter +asks `sys.sp_describe_undeclared_parameters`, which parses the batch and binds every name in it. + ### 2.2 Oracle — `bitween.db.oracle` | Setting | Default | Notes | @@ -424,6 +467,13 @@ It is best-effort by construction: the adapter has to be running on this node to source that is stopped or still starting cannot be asked, so the save proceeds unchecked. Refusing to let someone save a fix because the connection they are fixing it for is down would be backwards. +**Oracle's check is not the shared one either**, and for a worse reason: ODP.NET's `Prepare` is a +client-side no-op — Oracle compiles a statement when it is executed, not when it is prepared — so +the shared check passed everything, including a select from a table that does not exist. It uses +`DBMS_SQL.PARSE` instead, which compiles and resolves names while running nothing. The PL/SQL +frames that raises are stripped, so the answer is the one ORA- line that is about the operator's +SQL rather than the mechanism used to find it. + Two answers are not plain pass/fail. A bare **procedure name** passes with a `note` saying its existence was not checked — it is resolved when called, and preparing it as text is a syntax error every time. And a **wrong placeholder prefix** — `:name` on PostgreSQL, `@name` on Oracle — is diff --git a/tools/dev-database.md b/tools/dev-database.md index 97ce3852..0cd710a4 100644 --- a/tools/dev-database.md +++ b/tools/dev-database.md @@ -13,6 +13,45 @@ connect to something. order lines, an outbox that an integration drains, a set-returning function, a stored procedure and a sequence — with 60 orders and 30 unsent shipment notifications. +There is one per engine, and they are deliberately the same schema with the same rows, so a +statement written against one is recognisably the same job on another and the differences you meet +are the ones that are real: + +| File | Engine | What is different about it | +|---|---|---| +| `dev-warehouse.sql` | PostgreSQL | A set-returning function, because a PROCEDURE here cannot return rows. | +| `dev-warehouse-mysql.sql` | MySQL | A procedure that returns rows by SELECTing. No sequence — a counter table stands in. | +| `dev-warehouse-sqlserver.sql` | SQL Server | Everything in a `sales` schema; comments are extended properties; an inline table-valued function. | +| `dev-warehouse-oracle.sql` | Oracle | Parameters are `:name`; a REF CURSOR procedure; `fetch first` rather than `limit`. | + +```bash +# MySQL. --log-bin-trust-function-creators because creating a FUNCTION needs SUPER while binary +# logging is on, and the warehouse user is not SUPER. +docker run -d --name bw-mysql \ + -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=warehouse \ + -e MYSQL_USER=warehouse -e MYSQL_PASSWORD=warehouse \ + -p 55441:3306 mysql:8.4 --log-bin-trust-function-creators=1 +docker exec -i bw-mysql mysql -uwarehouse -pwarehouse warehouse < tools/dev-warehouse-mysql.sql + +# SQL Server. The 2022 image, not 2019: 2019 has no arm64 build and exits immediately on Apple +# silicon, which reads as "container is not running" rather than as anything about architecture. +docker run -d --name bw-mssql \ + -e ACCEPT_EULA=Y -e "MSSQL_SA_PASSWORD=Warehouse!2026" \ + -p 55442:1433 mcr.microsoft.com/mssql/server:2022-CU14-ubuntu-22.04 +docker exec bw-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Warehouse!2026' -C \ + -Q "create database warehouse" +docker cp tools/dev-warehouse-sqlserver.sql bw-mssql:/tmp/w.sql +docker exec bw-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Warehouse!2026' -C \ + -d warehouse -i /tmp/w.sql + +# Oracle. Slow to start — wait for "DATABASE IS READY TO USE" in the log before seeding. +docker run -d --name bw-oracle \ + -e ORACLE_PASSWORD=warehouse -e APP_USER=warehouse -e APP_USER_PASSWORD=warehouse \ + -p 55443:1521 gvenzl/oracle-free:23-slim-faststart +docker cp tools/dev-warehouse-oracle.sql bw-oracle:/tmp/w.sql +docker exec bw-oracle sqlplus -s warehouse/warehouse@localhost/FREEPDB1 @/tmp/w.sql +``` + ```bash docker run -d --name bw-sample-db \ -e POSTGRES_USER=warehouse -e POSTGRES_PASSWORD=warehouse -e POSTGRES_DB=warehouse \ diff --git a/tools/dev-warehouse-mysql.sql b/tools/dev-warehouse-mysql.sql new file mode 100644 index 00000000..587fd116 --- /dev/null +++ b/tools/dev-warehouse-mysql.sql @@ -0,0 +1,116 @@ +-- The same warehouse/order schema as dev-warehouse.sql, in MySQL's dialect. +-- +-- Deliberately the same shape, so a statement written against one engine is recognisably the same +-- job on another and the differences are the ones that are real: no sequences, a procedure that +-- returns rows by SELECTing, and `on duplicate key update` where PostgreSQL would write `merge`. + +create table customers ( + id int auto_increment primary key, + code varchar(20) not null unique, + name varchar(120) not null, + country char(2) not null, + credit_limit decimal(12,2) not null default 0 +) comment 'Trading partners we ship to'; + +create table orders ( + id int auto_increment primary key, + order_no varchar(30) not null unique, + customer_id int not null, + status varchar(20) not null default 'NEW', + total decimal(12,2) not null, + currency char(3) not null default 'USD', + placed_on timestamp not null default current_timestamp, + updated_on timestamp not null default current_timestamp on update current_timestamp, + constraint fk_orders_customer foreign key (customer_id) references customers (id) +) comment 'Sales orders, the table Bitween writes into'; + +create index ix_orders_status on orders (status); +create index ix_orders_updated on orders (updated_on); + +create table order_lines ( + id int auto_increment primary key, + order_id int not null, + sku varchar(40) not null, + quantity int not null, + unit_price decimal(12,2) not null, + constraint fk_lines_order foreign key (order_id) references orders (id) on delete cascade +); + +-- The outbox an integration polls: rows appear here, Bitween drains them. +create table shipment_outbox ( + id int auto_increment primary key, + order_no varchar(30) not null, + payload json not null, + created_on timestamp not null default current_timestamp, + processed tinyint(1) not null default 0, + processed_on timestamp null +) comment 'Shipment notifications waiting to be picked up'; + +create index ix_outbox_pending on shipment_outbox (processed, id); + +-- No sequence: MySQL has AUTO_INCREMENT, which belongs to a column rather than being an object of +-- its own. A statement that needs the next reference reads it from a counter table instead, which +-- is the usual stand-in and is why the adapter's supported-object list leaves sequences out. +create table shipment_ref ( + name varchar(40) primary key, + next_value bigint not null +); +insert into shipment_ref (name, next_value) values ('shipment', 5000); + +delimiter // + +-- Returns rows by SELECTing. This is the shape PostgreSQL needs a set-returning function for and +-- Oracle needs an explicit REF CURSOR for; here it is just a procedure. +create procedure orders_for_customer(in p_code varchar(20)) +begin + select o.order_no, o.status, o.total, o.placed_on + from orders o join customers c on c.id = o.customer_id + where c.code = p_code + order by o.placed_on desc; +end // + +-- A real procedure: called, returns nothing. +create procedure release_order(in p_order_no varchar(30)) +begin + update orders set status = 'RELEASED' where order_no = p_order_no; +end // + +-- And one that answers through an OUT parameter rather than a result set. +create procedure count_open_orders(in p_code varchar(20), out p_total int) +begin + select count(*) into p_total + from orders o join customers c on c.id = o.customer_id + where c.code = p_code and o.status in ('NEW', 'RELEASED'); +end // + +delimiter ; + +insert into customers (code, name, country, credit_limit) values + ('ACME', 'Acme Trading Co', 'JO', 50000), + ('GLOBEX', 'Globex Corporation', 'AE', 120000), + ('INITECH', 'Initech LLC', 'US', 25000), + ('UMBRA', 'Umbrella Logistics', 'DE', 80000); + +-- No generate_series in MySQL, so the rows come from a recursive CTE. cte_max_recursion_depth +-- defaults to 1000, which is comfortably above the 60 wanted here. +insert into orders (order_no, customer_id, status, total, currency, placed_on) +with recursive g (n) as (select 1 union all select n + 1 from g where n < 60) +select concat('SO-', lpad(n, 5, '0')), + 1 + (n % 4), + elt(1 + (n % 4), 'NEW', 'RELEASED', 'SHIPPED', 'INVOICED'), + round(rand() * 4000 + 100, 2), + elt(1 + (n % 3), 'USD', 'EUR', 'JOD'), + now() - interval n hour + from g; + +insert into order_lines (order_id, sku, quantity, unit_price) +with recursive l (n) as (select 1 union all select n + 1 from l where n < 3) +select o.id, concat('SKU-', lpad(((o.id * 7 + l.n) % 200), 4, '0')), + 1 + ((o.id + l.n) % 9), round(rand() * 300 + 5, 2) + from orders o cross join l; + +insert into shipment_outbox (order_no, payload) +select o.order_no, + json_object('orderNo', o.order_no, 'status', o.status, + 'total', o.total, 'currency', o.currency) + from orders o where o.status in ('SHIPPED', 'INVOICED'); diff --git a/tools/dev-warehouse-oracle.sql b/tools/dev-warehouse-oracle.sql new file mode 100644 index 00000000..c03afdba --- /dev/null +++ b/tools/dev-warehouse-oracle.sql @@ -0,0 +1,119 @@ +-- The same warehouse/order schema as dev-warehouse.sql, in Oracle's dialect. +-- +-- Deliberately the same shape, so a statement written against one engine is recognisably the same +-- job on another and the differences are the ones that are real: a parameter is :name, a sequence +-- is an object, a comment is its own statement, and a procedure hands back rows through an +-- explicit REF CURSOR rather than by SELECTing. + +create table customers ( + id number generated always as identity primary key, + code varchar2(20) not null unique, + name varchar2(120) not null, + country char(2) not null, + credit_limit number(12,2) default 0 not null +); + +comment on table customers is 'Trading partners we ship to'; + +create table orders ( + id number generated always as identity primary key, + order_no varchar2(30) not null unique, + customer_id number not null references customers (id), + status varchar2(20) default 'NEW' not null, + total number(12,2) not null, + currency char(3) default 'USD' not null, + placed_on timestamp with time zone default systimestamp not null, + updated_on timestamp with time zone default systimestamp not null +); + +comment on table orders is 'Sales orders, the table Bitween writes into'; + +create index ix_orders_status on orders (status); +create index ix_orders_updated on orders (updated_on); + +create table order_lines ( + id number generated always as identity primary key, + order_id number not null references orders (id) on delete cascade, + sku varchar2(40) not null, + quantity number not null, + unit_price number(12,2) not null +); + +-- The outbox an integration polls: rows appear here, Bitween drains them. +create table shipment_outbox ( + id number generated always as identity primary key, + order_no varchar2(30) not null, + payload clob not null, + created_on timestamp with time zone default systimestamp not null, + processed number(1) default 0 not null, + processed_on timestamp with time zone +); + +comment on table shipment_outbox is 'Shipment notifications waiting to be picked up'; + +create index ix_outbox_pending on shipment_outbox (processed, id); + +create sequence shipment_ref_seq start with 5000 increment by 1; + +insert into customers (code, name, country, credit_limit) values ('ACME', 'Acme Trading Co', 'JO', 50000); +insert into customers (code, name, country, credit_limit) values ('GLOBEX', 'Globex Corporation', 'AE', 120000); +insert into customers (code, name, country, credit_limit) values ('INITECH', 'Initech LLC', 'US', 25000); +insert into customers (code, name, country, credit_limit) values ('UMBRA', 'Umbrella Logistics', 'DE', 80000); + +insert into orders (order_no, customer_id, status, total, currency, placed_on) +select 'SO-' || lpad(level, 5, '0'), + 1 + mod(level, 4), + decode(mod(level, 4), 0, 'NEW', 1, 'RELEASED', 2, 'SHIPPED', 'INVOICED'), + round(dbms_random.value(100, 4100), 2), + decode(mod(level, 3), 0, 'USD', 1, 'EUR', 'JOD'), + systimestamp - numtodsinterval(level, 'HOUR') + from dual connect by level <= 60; + +insert into order_lines (order_id, sku, quantity, unit_price) +select o.id, + 'SKU-' || lpad(mod(o.id * 7 + l.n, 200), 4, '0'), + 1 + mod(o.id + l.n, 9), + round(dbms_random.value(5, 305), 2) + from orders o + cross join (select level as n from dual connect by level <= 3) l; + +insert into shipment_outbox (order_no, payload) +select o.order_no, + json_object('orderNo' value o.order_no, 'status' value o.status, + 'total' value o.total, 'currency' value o.currency) + from orders o + where o.status in ('SHIPPED', 'INVOICED'); + +commit; + +-- A REF CURSOR procedure: Oracle's way of handing rows back from a CALL, and the reason the +-- adapter's capability list says procedureResultSets is true here and false on PostgreSQL. +create or replace procedure orders_for_customer(p_code in varchar2, p_rows out sys_refcursor) +as +begin + open p_rows for + select o.order_no, o.status, o.total, o.placed_on + from orders o join customers c on c.id = o.customer_id + where c.code = p_code + order by o.placed_on desc; +end; +/ + +-- A real procedure: called, returns nothing. +create or replace procedure release_order(p_order_no in varchar2) +as +begin + update orders set status = 'RELEASED', updated_on = systimestamp where order_no = p_order_no; + commit; +end; +/ + +-- And one that answers through an OUT parameter rather than a cursor. +create or replace procedure count_open_orders(p_code in varchar2, p_total out number) +as +begin + select count(*) into p_total + from orders o join customers c on c.id = o.customer_id + where c.code = p_code and o.status in ('NEW', 'RELEASED'); +end; +/ diff --git a/tools/dev-warehouse-sqlserver.sql b/tools/dev-warehouse-sqlserver.sql new file mode 100644 index 00000000..d902ffe7 --- /dev/null +++ b/tools/dev-warehouse-sqlserver.sql @@ -0,0 +1,152 @@ +-- The same warehouse/order schema as dev-warehouse.sql, in T-SQL. +-- +-- Deliberately the same shape, so a statement written against one engine is recognisably the same +-- job on another and the differences are the ones that are real: everything lives in a named +-- schema, a comment is an extended property, and OUTPUT does what RETURNING does. + +create schema sales; +go + +create table sales.customers ( + id int identity(1,1) primary key, + code varchar(20) not null unique, + name nvarchar(120) not null, + country char(2) not null, + credit_limit decimal(12,2) not null constraint df_credit default 0 +); +go + +exec sys.sp_addextendedproperty + @name = N'MS_Description', @value = N'Trading partners we ship to', + @level0type = N'SCHEMA', @level0name = N'sales', + @level1type = N'TABLE', @level1name = N'customers'; +go + +create table sales.orders ( + id int identity(1,1) primary key, + order_no varchar(30) not null unique, + customer_id int not null references sales.customers (id), + status varchar(20) not null constraint df_status default 'NEW', + total decimal(12,2) not null, + currency char(3) not null constraint df_currency default 'USD', + placed_on datetime2(3) not null constraint df_placed default sysutcdatetime(), + updated_on datetime2(3) not null constraint df_updated default sysutcdatetime() +); +go + +exec sys.sp_addextendedproperty + @name = N'MS_Description', @value = N'Sales orders, the table Bitween writes into', + @level0type = N'SCHEMA', @level0name = N'sales', + @level1type = N'TABLE', @level1name = N'orders'; +go + +create index ix_orders_status on sales.orders (status); +create index ix_orders_updated on sales.orders (updated_on); +go + +create table sales.order_lines ( + id int identity(1,1) primary key, + order_id int not null references sales.orders (id) on delete cascade, + sku varchar(40) not null, + quantity int not null, + unit_price decimal(12,2) not null +); +go + +-- The outbox an integration polls: rows appear here, Bitween drains them. +create table sales.shipment_outbox ( + id int identity(1,1) primary key, + order_no varchar(30) not null, + payload nvarchar(max) not null, + created_on datetime2(3) not null constraint df_outbox_created default sysutcdatetime(), + processed bit not null constraint df_outbox_processed default 0, + processed_on datetime2(3) null +); +go + +exec sys.sp_addextendedproperty + @name = N'MS_Description', @value = N'Shipment notifications waiting to be picked up', + @level0type = N'SCHEMA', @level0name = N'sales', + @level1type = N'TABLE', @level1name = N'shipment_outbox'; +go + +create index ix_outbox_pending on sales.shipment_outbox (processed, id); +go + +-- A real sequence object, which MySQL does not have and PostgreSQL and Oracle do. +create sequence sales.shipment_ref_seq as bigint start with 5000 increment by 1; +go + +-- An inline table-valued function: the T-SQL answer to a set-returning function, and the shape a +-- receive statement would SELECT from. +create function sales.orders_for_customer(@p_code varchar(20)) +returns table +as +return ( + select o.order_no, o.status, o.total, o.placed_on + from sales.orders o join sales.customers c on c.id = o.customer_id + where c.code = @p_code +); +go + +-- A real procedure: called, returns nothing. +create procedure sales.release_order @p_order_no varchar(30) +as +begin + set nocount on; + update sales.orders + set status = 'RELEASED', updated_on = sysutcdatetime() + where order_no = @p_order_no; +end; +go + +-- And one that answers through an OUT parameter rather than a result set. +create procedure sales.count_open_orders @p_code varchar(20), @p_total int output +as +begin + set nocount on; + select @p_total = count(*) + from sales.orders o join sales.customers c on c.id = o.customer_id + where c.code = @p_code and o.status in ('NEW', 'RELEASED'); +end; +go + +insert into sales.customers (code, name, country, credit_limit) values + ('ACME', N'Acme Trading Co', 'JO', 50000), + ('GLOBEX', N'Globex Corporation', 'AE', 120000), + ('INITECH', N'Initech LLC', 'US', 25000), + ('UMBRA', N'Umbrella Logistics', 'DE', 80000); +go + +-- No generate_series before SQL Server 2022, and this has to run on 2019 too, so the rows come +-- from a recursive CTE. +with g (n) as ( + select 1 union all select n + 1 from g where n < 60 +) +insert into sales.orders (order_no, customer_id, status, total, currency, placed_on) +select 'SO-' + right('00000' + cast(n as varchar(5)), 5), + 1 + (n % 4), + choose(1 + (n % 4), 'NEW', 'RELEASED', 'SHIPPED', 'INVOICED'), + cast(abs(checksum(newid())) % 4000 + 100 as decimal(12,2)), + choose(1 + (n % 3), 'USD', 'EUR', 'JOD'), + dateadd(hour, -n, sysutcdatetime()) + from g +option (maxrecursion 100); +go + +with l (n) as (select 1 union all select n + 1 from l where n < 3) +insert into sales.order_lines (order_id, sku, quantity, unit_price) +select o.id, + 'SKU-' + right('0000' + cast(((o.id * 7 + l.n) % 200) as varchar(4)), 4), + 1 + ((o.id + l.n) % 9), + cast(abs(checksum(newid())) % 300 + 5 as decimal(12,2)) + from sales.orders o cross join l; +go + +insert into sales.shipment_outbox (order_no, payload) +select o.order_no, + (select o.order_no as orderNo, o.status, o.total, o.currency + for json path, without_array_wrapper) + from sales.orders o + where o.status in ('SHIPPED', 'INVOICED'); +go From 2e7509eda2931f28f066b5ab984d9ecb62569cb5 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Fri, 11 Sep 2026 08:59:28 +0300 Subject: [PATCH 2/3] docs: a per-engine guide, for someone who knows SQL but not this database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The settings tables said what each field is. They did not say which four you have to fill in, which defaults to leave alone, how the engine writes a placeholder or a row limit, what its types arrive as, whether a procedure can return rows, or what its error messages mean. That is the whole of what someone needs on the day they point Bitween at a database they have not met. One section per engine, each answering the same five questions, plus the parts that are the same everywhere, how to choose a receive mode, and a minimal grant per engine. The reference links to it and it links back. Writing it turned up two things that were not true: - PostgreSQL claimed MERGE whatever the server was. It arrived in 15, and 13 and 14 are both still widely run — so a capability list that said yes was a statement somebody writes and cannot parse. Capabilities can now be narrowed once the version is known, which is the only point at which the difference can be told, and the rule is unit-tested rather than needing a container per release. - A PostgreSQL table that has never been ANALYZEd reports reltuples as -1, and that was clamped to 0 — so the browser said "no rows" about a table that might hold millions. Null means unknown, and null is what it says now. And one thing the doc got wrong before the code was checked: Oracle reports every NUMBER as decimal, including NUMBER(10,0). The hint is coarse on purpose. 475 unit, 431 integration, 313 client tests. Co-Authored-By: Claude Opus 5 --- .../DbResidentAdapterBase.cs | 26 + .../PostgreSqlDbAdapter.cs | 31 +- .../CapabilityVersionTests.cs | 48 ++ .../SW.Bitween.UnitTests.csproj | 3 + docs/database-adapters-api.md | 6 + docs/database-adapters-per-engine.md | 465 ++++++++++++++++++ tools/dev-database.md | 4 + 7 files changed, 578 insertions(+), 5 deletions(-) create mode 100644 SW.Bitween.UnitTests/CapabilityVersionTests.cs create mode 100644 docs/database-adapters-per-engine.md diff --git a/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs b/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs index bd090c36..20240051 100644 --- a/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs +++ b/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs @@ -187,6 +187,27 @@ protected async Task OpenAsync(CancellationToken cancellationToken return connection; } + ///

+ /// Narrows the declared capability list to what THIS server actually has. + /// + /// The list is written per engine and is therefore about the engine at its newest. Called once + /// the server version is known, which is the only point at which the difference can be told. + /// Does nothing unless an engine overrides it — most capabilities have been there for a decade. + /// + protected virtual void AdjustForVersion(DbCapabilities described) { } + + /// + /// The major version this server reports, or 0 when it cannot be read. Drivers spell it + /// differently — "16.14", "8.4.11", "16.00.4135" — but all of them lead with the major. + /// + protected static int MajorVersionOf(string serverVersion) + { + if (string.IsNullOrWhiteSpace(serverVersion)) return 0; + + var lead = serverVersion.Split('.', ' ')[0]; + return int.TryParse(lead, out var major) ? major : 0; + } + /// /// Runs once on every connection this adapter opens, before anything uses it. /// @@ -362,6 +383,11 @@ public virtual async Task Describe() // and the prefix this adapter actually binds with cannot drift apart. described.ParameterPrefix = ParameterPrefix; + // A chance for an engine to correct what it declared once it knows which VERSION it is + // talking to. MERGE arrived in PostgreSQL 15 and sequences in SQL Server 2012; a list that + // says "yes" against an older server is a statement somebody writes and cannot run. + AdjustForVersion(described); + described.Details["statements"] = string.Join(", ", statements.Names.OrderBy(n => n)); described.Details["allowAdHocSql"] = Options.AllowAdHocSql.ToString(); return described; diff --git a/SW.Bitween.Adapters.Db.PostgreSql/PostgreSqlDbAdapter.cs b/SW.Bitween.Adapters.Db.PostgreSql/PostgreSqlDbAdapter.cs index 48cc942d..5241d997 100644 --- a/SW.Bitween.Adapters.Db.PostgreSql/PostgreSqlDbAdapter.cs +++ b/SW.Bitween.Adapters.Db.PostgreSql/PostgreSqlDbAdapter.cs @@ -130,6 +130,25 @@ protected override string BuildConnectionString() ReceiveModes = ["bulk", "incrementing", "timestamp", "timestamp+incrementing", "marker"] }; + /// + /// MERGE arrived in PostgreSQL 15. Claiming it against 13 or 14 — both still widely run, and + /// both supported until 2025 and 2026 — would have somebody write a statement the server + /// cannot parse, and find out on the first message rather than here. + /// + protected override void AdjustForVersion(DbCapabilities described) => Narrow(described); + + /// + /// The narrowing itself, reachable without an adapter instance. Proving this needs one server + /// per version otherwise, and the rule is a comparison rather than anything the database does. + /// + public static void AdjustForVersionForTests(DbCapabilities described) => Narrow(described); + + static void Narrow(DbCapabilities described) + { + var major = MajorVersionOf(described.ServerVersion); + if (major > 0 && major < 15) described.Merge = false; + } + /// /// What this ROLE may do, asked of the server rather than assumed. Role attributes and database /// privileges are separate things in PostgreSQL and both matter — REPLICATION in particular, @@ -259,12 +278,14 @@ and n.nspname not in ('pg_catalog', 'information_schema', 'pg_toast') Type = TypeOf(reader.GetChar(2)), Comment = reader.IsDBNull(3) ? null : reader.GetString(3), - // reltuples, so it is as fresh as the last ANALYZE, and -1 on a table that has - // never been analysed. Reported as an estimate everywhere it surfaces — the - // alternative is COUNT(*) on a stranger's table, which a menu should not do. - RowCount = !request.IncludeRowCounts || reader.IsDBNull(4) + // reltuples, so it is as fresh as the last ANALYZE. It is -1 on a table that + // has never been analysed, and that is UNKNOWN rather than empty — clamping it + // to zero said "no rows" about a table that may hold millions. Reported as an + // estimate everywhere it surfaces; the alternative is COUNT(*) on a stranger's + // table, which a menu should not do. + RowCount = !request.IncludeRowCounts || reader.IsDBNull(4) || reader.GetInt64(4) < 0 ? null - : Math.Max(0, reader.GetInt64(4)) + : reader.GetInt64(4) }); } diff --git a/SW.Bitween.UnitTests/CapabilityVersionTests.cs b/SW.Bitween.UnitTests/CapabilityVersionTests.cs new file mode 100644 index 00000000..8d44f8a6 --- /dev/null +++ b/SW.Bitween.UnitTests/CapabilityVersionTests.cs @@ -0,0 +1,48 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Adapters.Db; +using SW.Bitween.Adapters.Db.PostgreSql; + +namespace SW.Bitween.UnitTests; + +/// +/// A capability list is written per engine and is therefore about the engine at its NEWEST. Where +/// a feature arrived in a specific release, the list has to be narrowed once the server version is +/// known — otherwise it promises something an older server will refuse, and the promise is found +/// out on the first message rather than on the screen that made it. +/// +[TestClass] +public class CapabilityVersionTests +{ + [TestMethod] + public void Merge_is_claimed_on_PostgreSQL_15_and_later() + { + Assert.IsTrue(MergeFor("15.6")); + Assert.IsTrue(MergeFor("16.14")); + Assert.IsTrue(MergeFor("17.2")); + } + + [TestMethod] + public void Merge_is_withdrawn_on_PostgreSQL_before_15() + { + // 13 and 14 are both still widely run and were supported into 2025 and 2026. + Assert.IsFalse(MergeFor("13.14")); + Assert.IsFalse(MergeFor("14.11")); + } + + [TestMethod] + public void An_unreadable_version_leaves_the_declared_list_alone() + { + // Better to claim what the engine can do at its newest than to strip a capability because + // a driver reported the version in a shape this did not expect. + Assert.IsTrue(MergeFor(null)); + Assert.IsTrue(MergeFor("")); + Assert.IsTrue(MergeFor("not a version")); + } + + static bool MergeFor(string serverVersion) + { + var described = new DbCapabilities { Merge = true, ServerVersion = serverVersion }; + PostgreSqlDbAdapter.AdjustForVersionForTests(described); + return described.Merge; + } +} diff --git a/SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj b/SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj index cfd41424..a4cdf9b7 100644 --- a/SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj +++ b/SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj @@ -35,6 +35,9 @@ referencing it from a net10.0 test project is fine and keeps the pure logic — the statement allow-list, cursor formatting — testable without a database. --> + + diff --git a/docs/database-adapters-api.md b/docs/database-adapters-api.md index c864c710..c4357f35 100644 --- a/docs/database-adapters-api.md +++ b/docs/database-adapters-api.md @@ -41,6 +41,12 @@ Two rules run through everything below: The data source form is generated from the adapter, so this table is what you will see on screen. Settings marked **secret** are encrypted at rest and never returned by the API. +> **Meeting one of these engines for the first time?** +> [database-adapters-per-engine.md](database-adapters-per-engine.md) is the guide behind these +> tables: what you must set and what you can leave alone, how each engine writes a placeholder and +> a row limit, what its types arrive as, whether a procedure can return rows, and what its error +> messages mean. This section is the reference; that one is the explanation. + ### 2.1 Common to both engines | Setting | Default | What it is for | diff --git a/docs/database-adapters-per-engine.md b/docs/database-adapters-per-engine.md new file mode 100644 index 00000000..454ed276 --- /dev/null +++ b/docs/database-adapters-per-engine.md @@ -0,0 +1,465 @@ +# Connecting Bitween to a database: a guide per engine + +Written for a developer who knows SQL and has met a relational database before, but has not +necessarily met *this* one. It assumes nothing about Bitween beyond the fact that you are pointing +it at a database and want it to work. + +Each engine gets the same five questions answered: + +1. **What must I set**, and what happens if I leave the rest alone. +2. **How do I write a statement** — placeholders, row limits, identifier case. +3. **What does the database hand back**, and as what. +4. **How do I call a procedure**, and will it return rows. +5. **What will go wrong**, and what the message means when it does. + +The terse settings tables live in +[database-adapters-api.md §2](database-adapters-api.md). This is the explanation behind them. + +--- + +## The parts that are the same everywhere + +**A data source is a connection, held open.** The adapter is a long-lived process with an ADO.NET +pool inside it, so a connect, a TLS handshake and an authentication round trip are paid once rather +than per message. That is the whole reason it exists, and it is why the pool settings matter more +than they would in a web app: `MinPoolSize` connections are held **per node**, so multiply by your +replica count before comparing against the server's session limit. + +**SQL lives on the data source, not in a message.** You create *statements* — named pieces of SQL +belonging to the connection — and a subscription names one. A message supplies parameter *values* +and never SQL text. This is not ceremony: adapter property values have `{{partner.X}}` substituted +into them before the adapter sees them, so SQL in a subscription's properties would be steerable by +ordinary partner data. + +**A statement is checked when you save it.** The engine parses and plans it — never runs it — so a +typo is refused while you are still looking at the form. If the connection is not running at that +moment the save says *"Saved, but not checked"* rather than pretending. + +**Only the engine's own answers are reported.** The capability list (`Describe`) is what *this* +server can do, narrowed by version where it matters. If it says `merge: false`, writing a `MERGE` +will fail, and the list is telling you so first. + +--- + +## PostgreSQL — `bitween.db.postgresql` + +The easiest of the four to configure, and the one most likely to be right by default. + +### What you must set + +`Host`, `Database`, `UserName`, `Password`. That is genuinely all. + +`Database` is required and has no default, because **PostgreSQL connects to a database, not to a +server** — there is no "log in and then pick one". It is the name you would pass to `psql -d`. + +### What the defaults do + +| Setting | Default | Leave it unless… | +|---|---|---| +| `Port` | 5432 | | +| `SslMode` | `Prefer` | **Change this to `Require` for anything not on localhost.** `Prefer` will silently fall back to an unencrypted connection if the server does not offer TLS — which is precisely the case you wanted to be told about. | +| `Schema` | server default | You want unqualified names to resolve somewhere other than `public`. Sets `search_path`; takes a list, e.g. `sales, public`. | +| `ApplicationName` | `Bitween` | Keep it distinctive — it is how a DBA finds your connections in `pg_stat_activity`. | +| `AutoPrepare` | `false` | You are running the same statement constantly and have measured a win. It caches a plan per connection, and a plan built against one shape of data can be worse than replanning. | +| `ConnectionIdleLifetimeSeconds` | 300 | | + +### Writing a statement + +- **Parameters are `@name`.** *Not* `:name` — that collides with the `::` cast operator, and + `value::text` would be read as a parameter called `text`. +- **Row limit is `limit N`**, at the end. +- **Identifiers fold to lower case** unless quoted. A table created as `"Orders"` must always be + written `"Orders"`; one created as `Orders` is `orders` and can be written either way. + +```sql +select id, order_no, total + from sales.orders + where customer_id = @customerId + order by id + limit 100 +``` + +### What comes back + +`numeric` → decimal, `timestamptz` → DateTime, `uuid` → Guid, `jsonb` → a string, `bytea` → bytes, +arrays → an array. A row count from `Discover` is `reltuples`, which is as fresh as the last +`ANALYZE` and is **null on a table that has never been analysed** — that is an honest "unknown", +not a zero. + +### Procedures and functions + +This is the one thing PostgreSQL does differently from the other three, and it catches people: + +- **A `PROCEDURE` called with `CALL` cannot return a result set.** The capability list says + `procedureResultSets: false` for exactly this reason. +- **A set-returning `FUNCTION` can**, and it is queried with `SELECT`, not `CALL`: + + ```sql + select * from orders_by_customer(@code) + ``` + + So a "stored procedure that returns rows" from an Oracle or SQL Server background is a *function* + here, and it goes through Query rather than Call. + +### Receiving + +Any always-increasing column works as a cursor — a `serial`/`identity` id, or a `timestamptz` +updated on write. Use `where id > @cursor order by id`; the `order by` is not optional, because +without it the last row read is arbitrary and the cursor will skip rows. + +### What goes wrong + +| Message | What it means | +|---|---| +| `42P01: relation "x" does not exist` | Wrong name, wrong schema, or your `search_path` does not include it. Qualify it or set `Schema`. | +| `42703: column "x" does not exist` | Usually a case problem — the column was created quoted with capitals. | +| `syntax error at or near ":"` | You wrote `:name`. Use `@name`. The save-time check says this in words. | +| `merge: false` in the capability list | The server is older than 15. `MERGE` does not exist there; use `insert … on conflict`. | + +--- + +## MySQL — `bitween.db.mysql` + +Serves MariaDB too. Configuration is simple; the surprises are in what the engine does *not* have. + +### What you must set + +`Host`, `Database`, `UserName`, `Password`. + +**`Database` is also the schema.** MySQL has no separate concept — `CREATE SCHEMA` is a synonym for +`CREATE DATABASE` — so this one value is both what you connect to and what an unqualified table name +resolves against. There is no `search_path` equivalent and no separate `Schema` setting, and +browsing the catalog with no schema filter means *this database*, not every database on the server. + +### What the defaults do + +| Setting | Default | Leave it unless… | +|---|---|---| +| `Port` | 3306 | | +| `SslMode` | `Preferred` | **`Required` or above off localhost**, same reasoning as PostgreSQL's `Prefer`. | +| `ConnectionIdleLifetimeSeconds` | 180 | **Check your server's `wait_timeout`.** The server default is 28800, but a managed instance or a proxy in front of one is routinely far lower. A connection the *server* closed first surfaces as a broken pipe on the next message rather than as a timeout, so keep this comfortably below it. | +| `ServerPrepare` | `true` | You connect through ProxySQL or similar, where prepared statements pin a backend and defeat the pooling the proxy exists to provide. **Turning it off also weakens the save-time check** — with `IgnorePrepare` on, nothing is sent to the server and every statement reports valid. | +| `AllowZeroDateTime` | `false` | The schema contains `0000-00-00`, which MySQL permits and no .NET date type can hold. Only old schemas have these. | + +### Writing a statement + +- **Parameters are `@name`.** +- **Row limit is `limit N`**, at the end. +- **Identifier case follows the file system** on the server: case-sensitive on Linux, + case-insensitive on Windows and macOS. Write table names exactly as they were created and the + question never comes up. +- Backticks quote an identifier, not double quotes (unless `ANSI_QUOTES` is set). + +### What comes back + +| Column type | Arrives as | Worth knowing | +|---|---|---| +| `tinyint(1)` | **bool** | This *is* the boolean type — MySQL has no other. The driver returns a bool, so do not expect 0/1. | +| `bigint unsigned` | decimal | It does not fit in a signed 64-bit integer. | +| `int unsigned` | long | Same reason, one size up. | +| `decimal` | decimal | | +| `datetime`, `timestamp` | DateTime | `timestamp` is stored UTC and converted to the session time zone; `datetime` is not converted at all. | +| `json` | string | | +| `blob` family | bytes | | + +A row count from `Discover` is InnoDB's estimate from index statistics and can be out by a wide +margin on a table that has not been analysed. It is always null for a view. + +### Procedures and functions + +**A procedure returns rows simply by `SELECT`ing** — no cursor to declare, nothing to bind. This is +the capability PostgreSQL declares false, and it means the obvious thing works: + +```sql +create procedure orders_by_customer(in p_code varchar(20)) +begin + select * from orders where code = p_code; +end +``` + +A statement meant to be **called** holds the procedure's *name*, not SQL — `orders_by_customer`, +not `call orders_by_customer(...)`. The parameters are bound by the message. + +### What it does not have + +This is the part worth reading before you design against it: + +- **No sequences.** `AUTO_INCREMENT` belongs to a column, not to an object. If you need a shared + counter, a one-row table is the usual stand-in. +- **No `MERGE`.** The upsert is `insert … on duplicate key update`, which has different semantics. +- **No `RETURNING`.** (MariaDB has it for `INSERT` and `DELETE`; MySQL does not, and the capability + list says false so that a statement written against it fails here rather than in production.) +- **No array type.** A JSON array is the stand-in and arrives as a string. + +### Receiving + +An `AUTO_INCREMENT` id is the natural cursor. A `timestamp` column with +`on update current_timestamp` works for `timestamp` mode — but note that `timestamp` has +second resolution unless you declare `timestamp(3)` or finer, and two rows written in the same +second can straddle a poll boundary. Prefer the id where you have one. + +### What goes wrong + +| Message | What it means | +|---|---| +| `Table 'db.x' doesn't exist` | Wrong database, or a case mismatch on a Linux server. | +| `You have an error in your SQL syntax … near ':name'` | You wrote `:name`. Use `@name`. | +| `You do not have the SUPER privilege and binary logging is enabled` | You are trying to **create a function**, not to run one. The server needs `log_bin_trust_function_creators=1`, which is a DBA switch — Bitween never creates routines. | +| Broken pipe / "server has gone away" on the first message after a quiet period | `ConnectionIdleLifetimeSeconds` is above the server's `wait_timeout`. | + +--- + +## SQL Server — `bitween.db.sqlserver` + +Serves Azure SQL too. The configuration surprise here is encryption; the SQL surprise is `TOP`. + +### What you must set + +`Host`, `Database`, `UserName`, `Password` — and very likely `TrustServerCertificate`. + +**`Encrypt` defaults to `true`**, which is a change from the old `System.Data.SqlClient` everyone +learned on. The modern driver encrypts by default and then *validates the certificate*, and the +usual on-premises instance presents a self-signed one. That combination is why an instance that +worked for years starts failing the moment something is upgraded. + +- **Azure SQL**: leave both alone. The certificate is real and validates. +- **On-premises with a self-signed certificate**: set `TrustServerCertificate = true`, knowing that + it means an attacker between you and the server could present their own. Installing the + certificate properly is better; this is the pragmatic answer. + +**A named instance is addressed through `Host`, not `Port`.** Write `SERVER\SQLEXPRESS` and leave +the port alone — a named instance is resolved by the SQL Server Browser service, and supplying both +is an error the driver reports obscurely. + +### What the defaults do + +| Setting | Default | Leave it unless… | +|---|---|---| +| `Port` | 1433 | You are using a named instance, in which case it is ignored. | +| `Schema` | the login's own | You want unqualified names to resolve to something other than the login's default (usually `dbo`). | +| `MultipleActiveResultSets` | `false` | Something specifically needs it. It stops the connection being reset cleanly between uses, which is the opposite of what a shared pool wants. | +| `SnapshotIsolation` | `false` | You want readers not to block writers. Only has an effect where the database has snapshot isolation enabled. | + +**A note on `Schema`.** SQL Server has no `search_path`: the default schema is a property of the +*login*, not of the connection. Bitween applies the setting per connection with `EXECUTE AS`, and +**failure is deliberately not fatal** — impersonating a schema's owner is a privilege many service +accounts do not have. If it cannot, the connection still works and unqualified names resolve as they +would have. Qualify your names (`sales.orders`) and the question never arises. + +### Writing a statement + +- **Parameters are `@name`.** +- **Row limit is `top N`, and it goes *before* the column list** — not at the end. This is the + dialect difference people hit first: + + ```sql + select top 100 id, order_no, total + from sales.orders + where customer_id = @customerId + order by id + ``` + + `offset … fetch next … rows only` also works and is what paging uses, but it **requires an + `order by`**. +- **Schemas are real and worth using.** `dbo` is a default, not a rule. +- Square brackets quote an identifier: `[order]`. + +### What comes back + +| Column type | Arrives as | Worth knowing | +|---|---|---| +| `bit` | bool | | +| `decimal`, `money` | decimal | | +| `datetime2`, `datetime` | DateTime | Prefer `datetime2`; `datetime` has ~3ms resolution and a 1753 floor. | +| `datetimeoffset` | DateTimeOffset | | +| `uniqueidentifier` | Guid | | +| `nvarchar`, `varchar` | string | | +| `varbinary`, `rowversion` | bytes | | + +A declared length you see in `Discover` is **halved for `n`-types**, because `max_length` in the +catalog is in bytes and an `nvarchar` stores two per character — so `nvarchar(50)` reports 50, as +you would write it, not 100. + +Row counts come from the partition statistics, which the engine maintains — cheaper and closer to +true than most engines' estimates, but still an estimate. + +### Procedures and functions + +**A procedure returns rows by `SELECT`ing**, like MySQL. Put `set nocount on` at the top so the +"N rows affected" messages do not arrive as extra result sets. + +Three kinds of function exist and all are listed: scalar (`FN`), inline table-valued (`IF`) and +multi-statement table-valued (`TF`). A **table-valued** function is the thing to `SELECT` from: + +```sql +select * from sales.orders_for(@code) +``` + +A statement meant to be **called** holds the procedure's name — `sales.orders_by_customer`. + +### What it has that the others may not + +- **`MERGE`**, the real statement. +- **`OUTPUT`**, which does what `RETURNING` does — and does it for `MERGE` too: + + ```sql + insert into sales.orders (order_no, total) + output inserted.* + values (@orderNo, @total) + ``` +- **Snapshot isolation**, which is genuinely useful for a polling receiver: a long read does not + block the writers it is reading from. + +### Receiving + +An `identity` column is the natural cursor. `rowversion` is the SQL-Server-specific answer and is +strictly monotonic across the database — but it arrives as bytes, not a number, so it does not fit +the `incrementing` mode; use an identity or a `datetime2` column. + +### What goes wrong + +| Message | What it means | +|---|---| +| `A connection was successfully established … but then an error occurred during the login process` / certificate chain errors | `Encrypt=true` against a self-signed certificate. Set `TrustServerCertificate`. | +| `Invalid object name 'x'` | Wrong schema, most often — the login's default is not what you assumed. Qualify it. | +| `Incorrect syntax near ':'` | You wrote `:name`. Use `@name`. | +| `Invalid usage of the option NEXT in the FETCH statement` | `offset/fetch` without an `order by`. Add one, or use `top`. | +| `The multi-part identifier could not be bound` | Almost always a typo in an alias or a join. | + +--- + +## Oracle — `bitween.db.oracle` + +The most configuration of the four, and the most dialect to remember. Everything here is normal +Oracle; none of it is Bitween being awkward. + +### What you must set + +`Host`, **exactly one of `ServiceName` or `Sid`**, `UserName`, `Password`. + +Both or neither is refused with a message saying so rather than guessed at. `ServiceName` is what a +modern Oracle wants — it is what `lsnrctl services` lists, e.g. `FREEPDB1` or `ORCLPDB1`. `Sid` is +for an older instance that has no service name. + +For **RAC, Data Guard, or Autonomous Database**, use `ConnectDescriptor` instead of host/port/ +service: a full TNS descriptor or an Easy Connect string. Pair it with `WalletDirectory` (the +folder holding `cwallet.sso`) for mTLS or cloud wallets. + +### What the defaults do + +| Setting | Default | Leave it unless… | +|---|---|---| +| `Port` | 1521 | | +| `Schema` | the login's own | You are reading someone else's schema. Sets `CURRENT_SCHEMA`, so unqualified names resolve there. | +| `BindByName` | `true` | **Never turn this off.** With it off ODP.NET binds by *position*, so a statement using `:id` twice — or one whose parameters arrive in a different order than they appear — silently binds the wrong values. Silently. | +| `FetchSize` | 100 | Rows per round trip. Raise it for wide reads over a slow link. | +| `AsSysDba` | `false` | Almost never right for an integration login. | + +### Writing a statement + +- **Parameters are `:name`.** This is the one engine of the four that uses a colon, and it is the + most common mistake when SQL is copied between data sources. The save-time check names it + explicitly: *"write `:code` rather than `@code`"*. +- **Row limit is `fetch first N rows only`**, at the end (12c and later). + **Do not use `rownum` with an `order by`** — `rownum` is applied *before* the sort, so you get an + arbitrary N rows and then sort those. It is the classic Oracle trap. +- **Identifiers are stored UPPER CASE** unless they were created quoted. A table created as + `orders` is `ORDERS` in the catalog, and the schema browser will show it that way. Unquoted SQL + is case-insensitive, so `select * from orders` works regardless. +- Oracle has no `boolean` in SQL before 23c — a flag is `NUMBER(1)` or `CHAR(1)`. Compare with + `= 1` or `= 'Y'` accordingly. + +```sql +select id, order_no, total + from orders + where customer_id = :customerId + order by id + fetch first 100 rows only +``` + +### What comes back + +| Column type | Arrives as | Worth knowing | +|---|---|---| +| `NUMBER`, any precision | decimal | Every `NUMBER` is reported as decimal, including `NUMBER(10,0)`. The type hint is coarse on purpose — it tells a mapper author to expect a number rather than a string, and `NUMBER` genuinely has more range than any binary float. | +| `VARCHAR2`, `CLOB` | string | | +| `DATE` | DateTime | Oracle's `DATE` **includes a time**, unlike every other engine here. | +| `TIMESTAMP WITH TIME ZONE` | DateTime | | +| `RAW`, `BLOB` | bytes | | + +A row count from `Discover` is `ALL_TABLES.NUM_ROWS`, which is **null until statistics are +gathered** (`DBMS_STATS.GATHER_TABLE_STATS`). Null means unknown, not empty. + +### Procedures and functions + +**A procedure returns rows through an explicit `SYS_REFCURSOR` out parameter** — this is Oracle's +answer to "a procedure that returns a result set", and the adapter binds and reads it for you: + +```sql +create or replace procedure orders_for_customer(p_code in varchar2, p_rows out sys_refcursor) +as +begin + open p_rows for select * from orders where code = p_code; +end; +``` + +A statement meant to be **called** holds the procedure's name — `orders_for_customer`. Because a +name is not SQL, the save-time check accepts it with a note saying its existence was not verified; +it is resolved when called. + +### Receiving + +An `identity` column (12c+) or a sequence-backed id is the natural cursor. Note that **a sequence +does not guarantee gap-free or commit-ordered values** — two sessions can take 5 and 6 and commit in +the other order, so a poll between the commits can miss one. Where that matters, use a +`marker` column (a processed flag) rather than a cursor, which is what the `marker` receive mode is +for. + +### What goes wrong + +| Message | What it means | +|---|---| +| `ORA-00942: table or view does not exist` | Wrong name, wrong schema, or **no grant** — Oracle reports "does not exist" for an object you cannot see, which is the same message either way. Check `ALL_TAB_PRIVS` before assuming a typo. | +| `ORA-00936: missing expression` | Often `@name` where `:name` was meant. The check says so. | +| `ORA-01017: invalid username/password` | As it says. Note that passwords are case-sensitive from 11g. | +| `ORA-12514: service not registered with the listener` | `ServiceName` is wrong, or the database is still starting. | +| `ORA-01722: invalid number` | An implicit string-to-number conversion failed — usually a parameter bound as text against a `NUMBER` column. | + +--- + +## Choosing a receive mode + +Independent of engine, and the thing most worth getting right: + +| Mode | Finds new rows by | Needs | Use when | +|---|---|---|---| +| `incrementing` | an always-growing column | cursor column | There is an id. The default answer. | +| `timestamp` | a modified-at column | cursor column | Rows are updated as well as inserted and you want both. | +| `timestamp+incrementing` | both | cursor column | Rows share a timestamp and you need a tiebreak. | +| `marker` | a processed-flag column | a mark-processed statement | There is no reliable ordering, or commits arrive out of order. | +| `bulk` | re-reading everything | a mark-processed statement | The table is a queue that is emptied. | + +**None of them can see a `DELETE`.** That is a property of polling, not of this adapter. If rows +disappear and that matters, the source needs a soft delete or an outbox. + +Always `order by` the cursor column. Without it the "last row read" is whatever the engine happened +to return last, and the cursor will skip rows silently. + +--- + +## A minimal grant + +Bitween needs to read what it reads and write what it writes, plus enough catalog access for the +schema browser. It never creates objects. + +| Engine | Minimum | +|---|---| +| PostgreSQL | `connect` on the database, `usage` on the schema, `select`/`insert`/`update` on the tables used. Catalog views are readable by default. | +| MySQL | `select` (plus `insert`/`update` where it writes) on the database, and `execute` on any routine it calls. `information_schema` is filtered by grant automatically. | +| SQL Server | `connect` on the database, `select`/`insert`/`update` on the objects, `execute` on the procedures, and `view definition` if you want the schema browser to show things the login has no other permission on. | +| Oracle | `create session`, `select`/`insert`/`update` on the objects, `execute` on the routines. The `ALL_*` catalog views show only what the login is granted, which is why a missing grant reads as "does not exist". | + +Whatever you grant, press **Test connection**. It reports what it connected as, what the login is +actually allowed to do, and then parses every statement you have configured against the live schema +— which is the quickest way to find a missing grant, a dropped column, or a statement written in +another engine's dialect. diff --git a/tools/dev-database.md b/tools/dev-database.md index 0cd710a4..8aed333e 100644 --- a/tools/dev-database.md +++ b/tools/dev-database.md @@ -24,6 +24,10 @@ are the ones that are real: | `dev-warehouse-sqlserver.sql` | SQL Server | Everything in a `sales` schema; comments are extended properties; an inline table-valued function. | | `dev-warehouse-oracle.sql` | Oracle | Parameters are `:name`; a REF CURSOR procedure; `fetch first` rather than `limit`. | +Each of those differences is explained in +[docs/database-adapters-per-engine.md](../docs/database-adapters-per-engine.md), along with what to +configure for each engine and what its errors mean. + ```bash # MySQL. --log-bin-trust-function-creators because creating a FUNCTION needs SUPER while binary # logging is on, and the warehouse user is not SUPER. From f5b1ce9bb728e7f44f6dd860eae71ac790f7752a Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Fri, 11 Sep 2026 11:08:24 +0300 Subject: [PATCH 3/3] feat: a delivery can publish to the customer's broker, from any node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publish had existed in both bus adapters since they did, and nothing could reach it: Bitween's pipeline calls Handle on a handler, and neither class had one. So an integration could drain a customer's queue and had no way to answer on it. Both adapters now implement IInfolinkHandler and declare [AdapterKind("handler")], so a subscription's delivery stage can target a queue. Adding Handle was the smaller half. A broker data source is EXCLUSIVE — one node holds the connection so the queue is drained once — while a delivery runs on whichever node picked the message up. Publishing would therefore have failed on every node but the owner, which is to say almost always. Exclusivity is about consuming, not about connecting. When the owned instance is not on this node, the runtime opens a send-only connection instead: built from the data source's own settings, Consume=false, no Endpoints, its own pool key so it can never be confused with the consuming one. It sends and never subscribes, so nothing is processed twice. That is the same Consume=false the connection test already uses, for the same reason. The rented path also had to carry the call's properties. Startup values are the connection's; where to publish is the subscription's, and passing only the spec sent the message to an endpoint the connection had never heard of. Two bugs this turned up: - WriteBackHealthAsync keyed a dictionary by InstanceKey, which is unique for an exclusive instance and not for a POOLED one. Publish-only instances made that routine, and a duplicate key took the whole reconcile pass down. It now looks only at instances that ARE a data source. - Subscriptions/Get stopped projecting DataSourceId — dropped when the r10 merge took their version of the file, which predates the field. Nothing failed: the API answered null while the row held an id, so the UI read every bound subscription as unbound and saving one from that screen wrote the null back. A test now asserts the read, because a projection is only as good as the thing that notices a field missing from it. UI: a broker is offered in a delivery (never in a receiver — ingress comes through a bus gateway), with Publish to / Exchange / Routing key, and a warning when the endpoint is empty. The connection is subscription-wide, so choosing a broker where another stage needs a database now says so rather than saving something that fails on its first message. The SQL hint no longer names two engines out of four — it reports the prefix the adapter itself declares. Co-Authored-By: Claude Opus 5 --- .../RabbitBusHandler.cs | 61 +++++++- .../SW.Bitween.Adapters.Bus.RabbitMq.csproj | 1 + SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs | 50 +++++- SW.Bitween.Api/Resources/Subscriptions/Get.cs | 6 + .../Adapters/ResidentAdapterRuntime.cs | 106 +++++++++++-- .../DataSources/BusProviderSupervisor.cs | 16 +- .../Tests/ExternalBusGatewayTests.cs | 102 ++++++++++++ .../Tests/SqsBusGatewayTests.cs | 73 +++++++++ .../Tests/SubscriptionLifecycleTests.cs | 42 +++++ .../src/pages/data-sources/Statements.tsx | 18 ++- .../pages/subscriptions/SubscriptionPage.tsx | 20 ++- .../studio/DataSourceBinding.tsx | 147 ++++++++++++++++-- docs/external-bus-providers.md | 44 +++++- 13 files changed, 645 insertions(+), 41 deletions(-) diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs index 335f4139..dfcdfe80 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs @@ -1,7 +1,9 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using RabbitMQ.Client; +using Newtonsoft.Json; using RabbitMQ.Client.Events; +using SW.PrimitiveTypes; using SW.Serverless.Sdk; using SW.Serverless.Sdk.Resident; using System; @@ -24,8 +26,14 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; /// A host rejection becomes BasicNack(requeue: true), so a Bitween outage does not lose the /// customer's messages — it just stops draining their queue, which is the correct failure. /// +// Two roles, one package. "bus" is what makes it configurable as a broker connection; "handler" +// is what puts it in the delivery picker, because the same resident instance both consumes a +// customer's queue and publishes back to one. Declared rather than encoded in the id: reclassifying +// by rename would break every gateway that stores it. [AdapterKind("bus")] -public class RabbitBusHandler(IOptions options, ILogger logger) : IResidentAdapter +[AdapterKind("handler")] +public class RabbitBusHandler(IOptions options, ILogger logger) + : IResidentAdapter, IInfolinkHandler { private readonly RabbitOptions _options = options.Value; @@ -312,6 +320,57 @@ public Task Publish(PublishRequest request) return Task.FromResult(new { messageId, bytes = body.Length }); } + /// + /// Egress through the PIPELINE: a subscription's delivery stage, publishing the message it was + /// given to a queue on this broker. + /// + /// has existed since this adapter did, and nothing could reach it — + /// Bitween's pipeline calls Handle on a handler, and this class had none, so an operator + /// could consume from a customer's broker and had no way to answer on it. This is the two of + /// them joined up; the publishing itself is unchanged. + /// + /// Where to send is the SUBSCRIPTION's business, not the connection's: one instance serves + /// every gateway on this broker, so the endpoint travels with the call rather than with the + /// process. Deliberately NOT restricted to the endpoints the data source consumes — the common + /// case for egress is a queue Bitween does not drain. + /// + public Task Handle(XchangeFile xchangeFile) + { + var endpoint = _context?.ValueOf("Endpoint"); + var exchange = _context?.ValueOf("Exchange"); + + if (string.IsNullOrWhiteSpace(endpoint) && string.IsNullOrWhiteSpace(exchange)) + throw new InvalidOperationException( + "This delivery has no Endpoint and no Exchange, so there is nowhere to publish. Set " + + "Endpoint to a queue name, or Exchange (with an optional RoutingKey) to publish " + + "through an exchange."); + + return PublishAsFile(xchangeFile, endpoint, exchange); + } + + async Task PublishAsFile(XchangeFile xchangeFile, string endpoint, string exchange) + { + var receipt = await Publish(new PublishRequest + { + Endpoint = endpoint, + Exchange = exchange, + RoutingKey = _context?.ValueOf("RoutingKey"), + ContentType = _context?.ValueOf("ContentType"), + + // The exchange id, so a redelivery is recognisable as the same message on the far side. + // Publishers that set nothing leave the consumer no way to deduplicate, which is the + // complaint this adapter logs when it receives one. + MessageId = _context?.ValueOf("xchangeid"), + + Body = xchangeFile?.Data ?? "" + }); + + // The broker's receipt as the response, so what was sent and under which id is on the + // exchange rather than only in a log. + return new XchangeFile( + JsonConvert.SerializeObject(receipt), xchangeFile?.Filename); + } + /// The control the UI needs before a data source is saved. Staged, so a failure names the step. public Task TestConnection() { diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj index 78fb846f..d180c52a 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj +++ b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj @@ -10,6 +10,7 @@ +