From 7e835491260d0203ec9441eaa343189b72bdf828 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Sun, 6 Sep 2026 04:13:50 +0300 Subject: [PATCH 01/43] feat: external bus providers for BusGateway, via resident adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A BusGateway can now be fed by an external broker instead of the internal bus, through a resident serverless adapter. Nothing existing changes. BusGateway.DataSourceId is nullable and null still means the internal bus, so every gateway already in a database behaves exactly as before. The ExternalBusDataSources migration is additive: three columns and one table, no data migration. Shape external broker -> resident adapter (owns the connection) -> BusProviderEventSink (resolves the gateway, persists) -> XchangeService.SubmitFilterXchange -> filter, mapper, handler, auto-retry — unchanged -> ack returns -> adapter acknowledges its broker Past the sink, ingress from a broker and ingress from the API are the same thing; none of the pipeline is reimplemented. The adapter does not acknowledge its broker until Bitween has persisted, so a Bitween outage stops draining the customer's queue rather than losing their messages. Domain DataSource holds how to reach a system — endpoint, credentials, health — while the gateway keeps what the message MEANS. One data source serves many gateways, as one connection serves many queues. DataSourceKind declares Relational/Document/ObjectStore alongside Broker because the adapter contract is identical for them; only push-vs-poll differs. Runtime BusProviderSupervisor reconciles running adapters against active data sources and writes heartbeat health back to the row, so a broker that has gone away is visible without tailing logs. Off by default via BitweenOptions .BusProvidersEnabled — a broker connection is exclusive and node placement is not implemented yet, so every instance would otherwise fight for it. DataSource.OwnedByNode exists for that election to write into. Providers Adapters.Bus.RabbitMq someone else's RabbitMQ. autoAck:false, ack only after Bitween persists, nack-with-requeue on rejection. DeclareMode defaults to `assert` because silently creating queues on a customer's broker is not our call. Dedupe key is the broker message id, not the delivery tag — tags are per channel and reset on reconnect. Publish included, so egress works on external gateways even though the internal one lacks it. Adapters.Bus.Sqs SQS is polled and has no ack, only delete. Same contract: receive, persist, ONLY THEN DeleteMessage. A rejection resets visibility to 0 so it retries in seconds. TestConnection warns when the visibility timeout is short enough to redeliver mid-persist. This is also the transport for Amazon Selling Partner API notifications, which SP-API delivers by publishing to an SQS queue you own. UnwrapSellingPartnerNotification strips the envelope and promotes notificationType and metadata to headers, so a Bitween document schema need not carry Amazon's wrapper. SP-API request/response calls are ordinary HTTPS and belong in a mapper — this covers the push half only. Both live under the Adapters/Bus Providers solution folder and target net8.0 deliberately: adapters are separate processes and must stay runnable on hosts that predate r10's move to net10.0. Temporary SimplyWorks.Serverless and .Sdk package references are swapped for ProjectReferences into ../SW-Serverless while simplify9/SW-Serverless#108 and #109 await approval. Restore them once merged and published. This also pulled SW.Serverless.Sdk and .Contract into the solution file. Docs: docs/external-bus-providers.md, including what is deliberately not done — placement, the DataSource CRUD API and UI, secret protection at rest, and broker-backed integration tests. Co-Authored-By: Claude Opus 5 --- SW.Bitween.Adapters.Bus.RabbitMq/Program.cs | 14 + .../RabbitBusHandler.cs | 352 +++ .../RabbitOptions.cs | 39 + .../SW.Bitween.Adapters.Bus.RabbitMq.csproj | 14 + SW.Bitween.Adapters.Bus.Sqs/Program.cs | 14 + .../SW.Bitween.Adapters.Bus.Sqs.csproj | 12 + SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs | 410 +++ SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs | 37 + SW.Bitween.Api/Data/BitweenDbContext.cs | 23 + .../Domain/DataSources/DataSource.cs | 80 + SW.Bitween.Api/Domain/Gateway/BusGateway.cs | 22 + SW.Bitween.Api/SW.Bitween.Api.csproj | 6 + SW.Bitween.Api/Services/BitweenOptions.cs | 11 + .../DataSources/BusProviderEventSink.cs | 96 + .../DataSources/BusProviderSupervisor.cs | 200 ++ .../SW.Bitween.IntegrationTests.csproj | 4 +- ...6011231_ExternalBusDataSources.Designer.cs | 2553 +++++++++++++++++ .../20260906011231_ExternalBusDataSources.cs | 120 + .../BitweenDbContextModelSnapshot.cs | 98 + ...W.Bitween.SampleConfigurableAdapter.csproj | 2 +- .../SW.Bitween.SampleHandler.csproj | 2 +- .../SW.Bitween.SampleMapper.csproj | 2 +- .../SW.Bitween.SampleValidator.csproj | 2 +- SW.Bitween.Web/SW.Bitween.Web.csproj | 4 +- SW.Bitween.Web/Startup.cs | 15 + SW.Bitween.sln | 63 + docs/external-brokers-architecture.md | 1134 ++++++++ docs/external-bus-providers.md | 98 + docs/provider-plan-rabbitmq-kafka.md | 734 +++++ 29 files changed, 6155 insertions(+), 6 deletions(-) create mode 100644 SW.Bitween.Adapters.Bus.RabbitMq/Program.cs create mode 100644 SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs create mode 100644 SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs create mode 100644 SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj create mode 100644 SW.Bitween.Adapters.Bus.Sqs/Program.cs create mode 100644 SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj create mode 100644 SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs create mode 100644 SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs create mode 100644 SW.Bitween.Api/Domain/DataSources/DataSource.cs create mode 100644 SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs create mode 100644 SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs create mode 100644 SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.Designer.cs create mode 100644 SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.cs create mode 100644 docs/external-brokers-architecture.md create mode 100644 docs/external-bus-providers.md create mode 100644 docs/provider-plan-rabbitmq-kafka.md diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/Program.cs b/SW.Bitween.Adapters.Bus.RabbitMq/Program.cs new file mode 100644 index 00000000..641004a7 --- /dev/null +++ b/SW.Bitween.Adapters.Bus.RabbitMq/Program.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using SW.Serverless.Sdk.Hosting; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Bus.RabbitMq; + +static class Program +{ + static Task Main() => AdapterHost.CreateBuilder() + .ConfigureServices((configuration, services) => services.Configure(configuration)) + .Build() + .RunResidentAsync(); +} diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs new file mode 100644 index 00000000..a33809a7 --- /dev/null +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs @@ -0,0 +1,352 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using SW.Serverless.Sdk.Resident; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Bus.RabbitMq; + +/// +/// Bitween's external RabbitMQ bus provider — a broker that is NOT the internal one, owned by +/// someone else, whose queues Bitween consumes and publishes to. +/// +/// The ack ordering is the contract: +/// +/// delivery -> PublishAsync to the host -> host persists the Xchange -> ack returns +/// -> ONLY THEN BasicAck +/// +/// A host rejection becomes BasicNack(requeue: true), so a Bitween outage does not lose the +/// customer's messages — it just stops draining their queue, which is the correct failure. +/// +public class RabbitBusHandler : IResidentAdapter +{ + private readonly RabbitOptions _options; + private readonly ILogger _logger; + + private IAdapterContext _context; + private IConnection _connection; + private IModel _consumeChannel; + private IModel _publishChannel; + private CancellationTokenSource _stopping; + + private readonly List _endpoints = new(); + private readonly Dictionary _consumerTags = new(); + + private long _received, _acked, _nacked, _failed, _published; + private DateTimeOffset? _lastMessageOn; + private string _lastError; + private volatile string _state = "Starting"; + + public RabbitBusHandler(IOptions options, ILogger logger) + { + _options = options.Value; + _logger = logger; + } + + // ---------------------------------------------------------------- lifecycle + + public Task StartAsync(IAdapterContext context, CancellationToken cancellationToken) + { + _context = context; + _stopping = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + _endpoints.AddRange((_options.Endpoints ?? "") + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + var factory = new ConnectionFactory + { + HostName = _options.Host, + Port = _options.Port, + UserName = _options.UserName, + Password = _options.Password ?? "", + VirtualHost = _options.VirtualHost, + RequestedConnectionTimeout = TimeSpan.FromSeconds(15), + + // The supervisor owns restart policy — backoff, crash-loop quarantine, health + // write-back. A second, hidden recovery loop in here would fight it. + AutomaticRecoveryEnabled = false + }; + if (_options.UseSsl) factory.Ssl = new SslOption { Enabled = true, ServerName = _options.Host }; + + _connection = factory.CreateConnection($"bitween-{context.InstanceKey}"); + _connection.ConnectionShutdown += (_, e) => + { + _state = "Disconnected"; + _lastError = $"{e.ReplyCode} {e.ReplyText}"; + _logger.LogWarning("Connection to {Host} closed: {Reason}", _options.Host, e.ReplyText); + }; + + _publishChannel = _connection.CreateModel(); + _consumeChannel = _connection.CreateModel(); + _consumeChannel.BasicQos(0, _options.Prefetch, global: false); + + DeclareTopology(_consumeChannel); + + foreach (var endpoint in _endpoints) + { + var consumer = new EventingBasicConsumer(_consumeChannel); + consumer.Received += (_, delivery) => _ = Task.Run(() => HandleAsync(endpoint, delivery)); + + // autoAck: false is what makes persist-then-ack possible at all. + _consumerTags[endpoint] = _consumeChannel.BasicConsume(endpoint, autoAck: false, consumer); + } + + _state = _endpoints.Count == 0 ? "Idle" : "Connected"; + _logger.LogInformation("Connected to {Host}:{Port}{VHost}, consuming {Count} endpoint(s) with prefetch {Prefetch}.", + _options.Host, _options.Port, _options.VirtualHost, _endpoints.Count, _options.Prefetch); + + return Task.CompletedTask; + } + + private void DeclareTopology(IModel channel) + { + var mode = (_options.DeclareMode ?? "assert").ToLowerInvariant(); + if (mode == "none") return; + + if (mode == "create" && !string.IsNullOrWhiteSpace(_options.Exchange)) + channel.ExchangeDeclare(_options.Exchange, _options.ExchangeType ?? "topic", + durable: _options.Durable, autoDelete: false); + + foreach (var endpoint in _endpoints) + { + if (mode == "assert") + { + // Throws if it does not exist, which is what we want: better to fail at start than + // to silently create a queue on someone else's broker. + channel.QueueDeclarePassive(endpoint); + continue; + } + + var arguments = new Dictionary(); + if (!string.IsNullOrWhiteSpace(_options.QueueType)) arguments["x-queue-type"] = _options.QueueType; + + channel.QueueDeclare(endpoint, durable: _options.Durable, exclusive: false, + autoDelete: false, arguments: arguments.Count == 0 ? null : arguments); + + if (!string.IsNullOrWhiteSpace(_options.Exchange)) + channel.QueueBind(endpoint, _options.Exchange, _options.RoutingKey ?? endpoint); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _state = "Draining"; + _stopping?.Cancel(); + + foreach (var tag in _consumerTags.Values) + try { _consumeChannel?.BasicCancel(tag); } catch { } + + try { _consumeChannel?.Close(); _publishChannel?.Close(); } catch { } + try { _connection?.Close(TimeSpan.FromSeconds(3)); } catch { } + _connection?.Dispose(); + + _state = "Stopped"; + return Task.CompletedTask; + } + + public Task GetStatusAsync() + { + var status = new AdapterStatus + { + Connected = _connection?.IsOpen == true, + State = _connection?.IsOpen != true ? "Disconnected" + : _received == 0 ? "Idle" : _state, + LastMessageOn = _lastMessageOn, + LastError = _lastError, + InFlight = Math.Max(0, _received - _acked - _nacked - _failed) + }; + + status.Details["host"] = $"{_options.Host}:{_options.Port}{_options.VirtualHost}"; + status.Details["endpoints"] = string.Join(",", _endpoints); + status.Details["prefetch"] = _options.Prefetch.ToString(); + status.Details["received"] = _received.ToString(); + status.Details["acked"] = _acked.ToString(); + status.Details["nacked"] = _nacked.ToString(); + status.Details["failed"] = _failed.ToString(); + status.Details["published"] = _published.ToString(); + + foreach (var endpoint in _endpoints) + status.Details[$"depth:{endpoint}"] = Depth(endpoint)?.ToString() ?? "?"; + + return Task.FromResult(status); + } + + private uint? Depth(string queue) + { + try + { + using var probe = _connection.CreateModel(); + return probe.QueueDeclarePassive(queue).MessageCount; + } + catch { return null; } + } + + // ---------------------------------------------------------------- ingress + + private async Task HandleAsync(string endpoint, BasicDeliverEventArgs delivery) + { + Interlocked.Increment(ref _received); + + try + { + var headers = new Dictionary + { + ["rabbit.exchange"] = delivery.Exchange, + ["rabbit.routingKey"] = delivery.RoutingKey, + ["rabbit.redelivered"] = delivery.Redelivered.ToString() + }; + if (delivery.BasicProperties?.MessageId is { Length: > 0 } messageId) + headers["rabbit.messageId"] = messageId; + + var result = await _context.PublishAsync( + delivery.Body, + // The broker's own message id when it has one, otherwise a content hash. NOT the + // delivery tag: tags are per channel and restart at 1 on every reconnect. + dedupeKey: delivery.BasicProperties?.MessageId is { Length: > 0 } id + ? $"rabbit:{_options.Host}:{endpoint}:{id}" + : $"rabbit:{_options.Host}:{endpoint}:{Convert.ToHexString(SHA256.HashData(delivery.Body.Span))[..32]}", + endpoint: endpoint, + headers: headers, + contentType: delivery.BasicProperties?.ContentType ?? "application/json", + cancellationToken: _stopping.Token); + + if (result.Accepted) + { + _consumeChannel.BasicAck(delivery.DeliveryTag, multiple: false); + Interlocked.Increment(ref _acked); + _lastMessageOn = DateTimeOffset.UtcNow; + _context.Metric("bitween.bus.rabbitmq.acked", 1); + } + else + { + _consumeChannel.BasicNack(delivery.DeliveryTag, multiple: false, requeue: true); + Interlocked.Increment(ref _nacked); + _lastError = result.Error; + _logger.LogWarning("Bitween rejected a message from {Endpoint}: {Error}. Requeued.", + endpoint, result.Error); + } + } + catch (OperationCanceledException) + { + try { _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } + } + catch (Exception ex) + { + Interlocked.Increment(ref _failed); + _lastError = ex.Message; + _logger.LogError(ex, "Failed to hand a delivery from {Endpoint} to Bitween.", endpoint); + try { _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } + } + } + + // ---------------------------------------------------------------- commands + + /// + /// Egress. Bitween does not have this on the internal gateway yet; an external provider gets + /// it for free because the adapter owns the connection either way. + /// + public Task Publish(PublishRequest request) + { + if (string.IsNullOrWhiteSpace(request?.Endpoint) && string.IsNullOrWhiteSpace(request?.Exchange)) + throw new ArgumentException("Either Endpoint or Exchange is required."); + + var properties = _publishChannel.CreateBasicProperties(); + properties.ContentType = request.ContentType ?? "application/json"; + properties.MessageId = request.MessageId ?? Guid.NewGuid().ToString("N"); + properties.DeliveryMode = (byte)(_options.Durable ? 2 : 1); + + var body = System.Text.Encoding.UTF8.GetBytes(request.Body ?? ""); + + lock (_publishChannel) + _publishChannel.BasicPublish( + exchange: request.Exchange ?? "", + routingKey: request.Exchange == null ? request.Endpoint : request.RoutingKey ?? "", + mandatory: false, + basicProperties: properties, + body: body); + + Interlocked.Increment(ref _published); + return Task.FromResult(new { messageId = properties.MessageId, bytes = body.Length }); + } + + /// The control the UI needs before a data source is saved. Staged, so a failure names the step. + public Task TestConnection() + { + var steps = new List(); + IConnection probe = null; + try + { + var factory = new ConnectionFactory + { + HostName = _options.Host, Port = _options.Port, + UserName = _options.UserName, Password = _options.Password ?? "", + VirtualHost = _options.VirtualHost, + RequestedConnectionTimeout = TimeSpan.FromSeconds(10) + }; + if (_options.UseSsl) factory.Ssl = new SslOption { Enabled = true, ServerName = _options.Host }; + + probe = factory.CreateConnection("bitween-probe"); + steps.Add(new { step = "connect", ok = true, detail = probe.Endpoint.ToString() }); + + using var channel = probe.CreateModel(); + steps.Add(new { step = "authenticate", ok = true, detail = _options.VirtualHost }); + + foreach (var endpoint in _endpoints) + { + try + { + var declared = channel.QueueDeclarePassive(endpoint); + steps.Add(new { step = $"queue:{endpoint}", ok = true, detail = $"{declared.MessageCount} message(s)" }); + } + catch (Exception ex) + { + steps.Add(new { step = $"queue:{endpoint}", ok = false, detail = ex.Message }); + return Task.FromResult(new { ok = false, steps }); + } + } + + return Task.FromResult(new { ok = true, steps }); + } + catch (Exception ex) + { + steps.Add(new { step = "failed", ok = false, detail = ex.Message }); + return Task.FromResult(new { ok = false, steps }); + } + finally + { + try { probe?.Close(); probe?.Dispose(); } catch { } + } + } + + /// What is actually on the broker, for the "pick a queue" step in the UI. + public Task Discover() => Task.FromResult(new + { + host = $"{_options.Host}:{_options.Port}", + virtualHost = _options.VirtualHost, + endpoints = _endpoints.Select(e => new { name = e, messages = Depth(e), consuming = _consumerTags.ContainsKey(e) }), + note = "AMQP alone can only report on queues we were told about. " + + "Listing everything on the broker needs the management plugin." + }); + + public Task GetStats() => Task.FromResult(new + { + received = _received, acked = _acked, nacked = _nacked, failed = _failed, published = _published, + endpoints = _endpoints, prefetch = _options.Prefetch + }); + + public class PublishRequest + { + public string Endpoint { get; set; } + public string Exchange { get; set; } + public string RoutingKey { get; set; } + public string MessageId { get; set; } + public string ContentType { get; set; } + public string Body { get; set; } + } +} diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs new file mode 100644 index 00000000..db2120f8 --- /dev/null +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs @@ -0,0 +1,39 @@ +namespace SW.Bitween.Adapters.Bus.RabbitMq; + +/// +/// Every one of these arrives as a DataSource property, bound by name. Nothing here is opinionated +/// about how the broker should be laid out: the queue may already exist and be owned by someone +/// else, or Bitween may declare it — decides which. +/// +public class RabbitOptions +{ + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 5672; + public string UserName { get; set; } = "guest"; + public string Password { get; set; } + public string VirtualHost { get; set; } = "/"; + public bool UseSsl { get; set; } + + /// Comma-separated queue names, supplied by the gateways bound to this data source. + public string Endpoints { get; set; } + + /// + /// none — assume everything exists; never touch the topology. + /// assert — verify it exists and fail loudly if not (passive declare). + /// create — declare queues, and an exchange and binding when Exchange is set. + /// + public string DeclareMode { get; set; } = "assert"; + + /// Optional. When set, each endpoint queue is bound to it. + public string Exchange { get; set; } + public string ExchangeType { get; set; } = "topic"; + public string RoutingKey { get; set; } + + /// Broker-side backpressure; pairs with the host's credit window. + public ushort Prefetch { get; set; } = 16; + + public bool Durable { get; set; } = true; + + /// Passed straight through as x-queue-type: classic, quorum or stream. + public string QueueType { get; set; } +} diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj new file mode 100644 index 00000000..5661e18b --- /dev/null +++ b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj @@ -0,0 +1,14 @@ + + + Exe + + net8.0 + disable + SW.Bitween.Adapters.Bus.RabbitMq + + + + + + diff --git a/SW.Bitween.Adapters.Bus.Sqs/Program.cs b/SW.Bitween.Adapters.Bus.Sqs/Program.cs new file mode 100644 index 00000000..9bd67630 --- /dev/null +++ b/SW.Bitween.Adapters.Bus.Sqs/Program.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using SW.Serverless.Sdk.Hosting; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Bus.Sqs; + +static class Program +{ + static Task Main() => AdapterHost.CreateBuilder() + .ConfigureServices((configuration, services) => services.Configure(configuration)) + .Build() + .RunResidentAsync(); +} diff --git a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj new file mode 100644 index 00000000..32fe9c2a --- /dev/null +++ b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj @@ -0,0 +1,12 @@ + + + Exe + net8.0 + disable + SW.Bitween.Adapters.Bus.Sqs + + + + + + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs new file mode 100644 index 00000000..873187b7 --- /dev/null +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs @@ -0,0 +1,410 @@ +using Amazon; +using Amazon.Runtime; +using Amazon.SQS; +using Amazon.SQS.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json.Linq; +using SW.Serverless.Sdk.Resident; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters.Bus.Sqs; + +/// +/// Amazon SQS as a Bitween bus provider. +/// +/// SQS is a poll-based queue, not a push broker, so the shape differs from RabbitMQ in one way +/// that matters: there is no ack, only DELETE. A message stays invisible for the visibility +/// timeout and reappears if it is not deleted — which is the same at-least-once contract by a +/// different mechanism, and maps cleanly onto persist-then-acknowledge: +/// +/// receive -> PublishAsync to the host -> host persists -> ONLY THEN DeleteMessage +/// +/// Reject or crash in between and SQS redelivers when the visibility timeout expires. So +/// VisibilityTimeoutSeconds must exceed how long Bitween takes to persist, or a message is +/// redelivered while the first copy is still being handled. +/// +/// WHY THIS ONE MATTERS BEYOND SQS ITSELF: the Amazon Selling Partner API delivers notifications +/// by publishing to an SQS queue that you own. You create the queue, grant SP-API permission to +/// send to it, then subscribe notification types to that destination. So this adapter is the +/// transport for SP-API notifications — ORDER_CHANGE, LISTINGS_ITEM_STATUS_CHANGE, REPORT_PROCESSING_FINISHED +/// and the rest — and handles the +/// envelope they arrive in. The SP-API request/response calls themselves are ordinary HTTPS and +/// belong in a mapper or handler, not here. +/// +public class SqsBusHandler : IResidentAdapter +{ + private readonly SqsOptions _options; + private readonly ILogger _logger; + + private IAmazonSQS _sqs; + private IAdapterContext _context; + private CancellationTokenSource _stopping; + private readonly List _pollers = new(); + private readonly List _endpoints = new(); + + private long _received, _deleted, _returned, _failed, _sent; + private DateTimeOffset? _lastMessageOn; + private string _lastError; + private volatile string _state = "Starting"; + + public SqsBusHandler(IOptions options, ILogger logger) + { + _options = options.Value; + _logger = logger; + } + + // ---------------------------------------------------------------- lifecycle + + public Task StartAsync(IAdapterContext context, CancellationToken cancellationToken) + { + _context = context; + _stopping = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + _endpoints.AddRange((_options.Endpoints ?? "") + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + _sqs = CreateClient(_options); + + _state = _endpoints.Count == 0 ? "Idle" : "Connected"; + _logger.LogInformation("SQS client for {Region}, polling {Count} queue(s) with {Wait}s long poll.", + _options.Region, _endpoints.Count, _options.WaitTimeSeconds); + + // One poller per queue. Long polling means these are cheap: a blocked receive costs + // nothing until a message arrives or the wait expires. + foreach (var endpoint in _endpoints) + _pollers.Add(Task.Run(() => PollAsync(endpoint, _stopping.Token))); + + return Task.CompletedTask; + } + + private static IAmazonSQS CreateClient(SqsOptions options) + { + var config = new AmazonSQSConfig { RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region) }; + + if (!string.IsNullOrWhiteSpace(options.ServiceUrl)) + { + config.ServiceURL = options.ServiceUrl; + config.AuthenticationRegion = options.Region; + } + + // No keys means the ambient chain — instance profile, IRSA, environment. That is the + // correct production answer on AWS, and explicit keys are the exception. + return string.IsNullOrWhiteSpace(options.AccessKeyId) + ? new AmazonSQSClient(config) + : new AmazonSQSClient( + new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey), config); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + _state = "Draining"; + _stopping?.Cancel(); + + // In-flight messages are simply not deleted, so SQS redelivers them after the visibility + // timeout. Nothing is lost by stopping mid-batch. + if (_pollers.Count > 0) + await Task.WhenAny(Task.WhenAll(_pollers), Task.Delay(5000, cancellationToken)); + + _sqs?.Dispose(); + _state = "Stopped"; + } + + public async Task GetStatusAsync() + { + var status = new AdapterStatus + { + Connected = _sqs != null && _state != "Disconnected", + State = _received == 0 && _state == "Connected" ? "Idle" : _state, + LastMessageOn = _lastMessageOn, + LastError = _lastError, + InFlight = Math.Max(0, _received - _deleted - _returned - _failed) + }; + + status.Details["region"] = _options.Region; + status.Details["endpoints"] = string.Join(",", _endpoints); + status.Details["visibilityTimeout"] = _options.VisibilityTimeoutSeconds.ToString(); + status.Details["received"] = _received.ToString(); + status.Details["deleted"] = _deleted.ToString(); + status.Details["returned"] = _returned.ToString(); + status.Details["failed"] = _failed.ToString(); + status.Details["sent"] = _sent.ToString(); + + // ApproximateNumberOfMessages is the SQS equivalent of queue depth, and the only backlog + // signal available without CloudWatch. + foreach (var endpoint in _endpoints) + { + var depth = await DepthAsync(endpoint); + if (depth != null) status.Details[$"depth:{Short(endpoint)}"] = depth; + } + + return status; + } + + private static string Short(string queueUrl) => queueUrl[(queueUrl.LastIndexOf('/') + 1)..]; + + private async Task DepthAsync(string queueUrl) + { + try + { + var attributes = await _sqs.GetQueueAttributesAsync(queueUrl, + new List { "ApproximateNumberOfMessages" }); + return attributes.ApproximateNumberOfMessages.ToString(); + } + catch { return null; } + } + + // ---------------------------------------------------------------- ingress + + private async Task PollAsync(string queueUrl, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var response = await _sqs.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = queueUrl, + MaxNumberOfMessages = Math.Clamp(_options.MaxMessagesPerReceive, 1, 10), + WaitTimeSeconds = Math.Clamp(_options.WaitTimeSeconds, 0, 20), + VisibilityTimeout = _options.VisibilityTimeoutSeconds, + MessageAttributeNames = new List { "All" }, + MessageSystemAttributeNames = new List { "All" } + }, ct); + + _state = "Connected"; + + foreach (var message in response.Messages ?? new List()) + { + if (ct.IsCancellationRequested) return; + await HandleAsync(queueUrl, message, ct); + } + } + catch (OperationCanceledException) { return; } + catch (Exception ex) + { + _state = "Disconnected"; + _lastError = ex.Message; + _logger.LogError(ex, "Polling {Queue} failed.", Short(queueUrl)); + + // Back off rather than hammering a failing endpoint; the supervisor decides + // whether this is terminal. + try { await Task.Delay(TimeSpan.FromSeconds(5), ct); } + catch (OperationCanceledException) { return; } + } + } + } + + private async Task HandleAsync(string queueUrl, Message message, CancellationToken ct) + { + Interlocked.Increment(ref _received); + + try + { + var headers = new Dictionary + { + ["sqs.messageId"] = message.MessageId, + ["sqs.queue"] = Short(queueUrl) + }; + + foreach (var attribute in message.MessageAttributes ?? new Dictionary()) + headers[$"sqs.attr.{attribute.Key}"] = attribute.Value?.StringValue ?? ""; + + var body = message.Body ?? ""; + + if (_options.UnwrapSellingPartnerNotification) + body = UnwrapSpApi(body, headers); + + var result = await _context.PublishAsync( + System.Text.Encoding.UTF8.GetBytes(body), + // MessageId is unique per message but NOT stable across redelivery of the same + // logical message on a standard queue, so a SequenceNumber or the SP-API + // notification id is preferred where present. + dedupeKey: headers.TryGetValue("spapi.notificationId", out var notificationId) + ? $"spapi:{notificationId}" + : $"sqs:{Short(queueUrl)}:{message.MessageId}", + endpoint: queueUrl, + headers: headers, + contentType: "application/json", + cancellationToken: ct); + + if (result.Accepted) + { + // The SQS equivalent of an ack. Until this call, the message is merely invisible. + await _sqs.DeleteMessageAsync(queueUrl, message.ReceiptHandle, ct); + Interlocked.Increment(ref _deleted); + _lastMessageOn = DateTimeOffset.UtcNow; + _context.Metric("bitween.bus.sqs.deleted", 1); + } + else + { + // Make it visible again immediately instead of waiting out the timeout, so a + // transient Bitween failure retries in seconds rather than minutes. + await ReturnToQueueAsync(queueUrl, message, ct); + Interlocked.Increment(ref _returned); + _lastError = result.Error; + _logger.LogWarning("Bitween rejected {MessageId} from {Queue}: {Error}. Returned to the queue.", + message.MessageId, Short(queueUrl), result.Error); + } + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + Interlocked.Increment(ref _failed); + _lastError = ex.Message; + _logger.LogError(ex, "Failed to hand {MessageId} to Bitween.", message.MessageId); + try { await ReturnToQueueAsync(queueUrl, message, CancellationToken.None); } catch { } + } + } + + private Task ReturnToQueueAsync(string queueUrl, Message message, CancellationToken ct) => + _sqs.ChangeMessageVisibilityAsync(queueUrl, message.ReceiptHandle, 0, ct); + + /// + /// SP-API notifications arrive wrapped: notificationType, notificationVersion, payloadVersion, + /// eventTime, notificationMetadata and the actual payload. Forwarding the whole envelope would + /// make every Bitween document schema carry Amazon's wrapper, so promote the metadata to + /// headers and pass the payload through. + /// + private string UnwrapSpApi(string body, IDictionary headers) + { + try + { + var envelope = JObject.Parse(body); + var payload = envelope["payload"]; + if (payload == null) return body; + + if (envelope["notificationType"]?.ToString() is { Length: > 0 } type) + headers["spapi.notificationType"] = type; + if (envelope["eventTime"]?.ToString() is { Length: > 0 } eventTime) + headers["spapi.eventTime"] = eventTime; + + var metadata = envelope["notificationMetadata"]; + if (metadata?["notificationId"]?.ToString() is { Length: > 0 } notificationId) + headers["spapi.notificationId"] = notificationId; + if (metadata?["subscriptionId"]?.ToString() is { Length: > 0 } subscriptionId) + headers["spapi.subscriptionId"] = subscriptionId; + + return payload.ToString(Newtonsoft.Json.Formatting.None); + } + catch (Exception ex) + { + // Not fatal: forward the raw body and let a mapper deal with it. + _logger.LogWarning(ex, "Body did not look like an SP-API notification; forwarding it whole."); + return body; + } + } + + // ---------------------------------------------------------------- commands + + public async Task Publish(PublishRequest request) + { + if (string.IsNullOrWhiteSpace(request?.Endpoint)) + throw new ArgumentException("Endpoint (the queue URL) is required."); + + var send = new SendMessageRequest { QueueUrl = request.Endpoint, MessageBody = request.Body ?? "" }; + + // FIFO queues require a group id, and reject the request without one. + if (request.Endpoint.EndsWith(".fifo", StringComparison.OrdinalIgnoreCase)) + { + send.MessageGroupId = request.GroupId ?? "bitween"; + if (!string.IsNullOrWhiteSpace(request.DeduplicationId)) + send.MessageDeduplicationId = request.DeduplicationId; + } + + var response = await _sqs.SendMessageAsync(send, _stopping.Token); + Interlocked.Increment(ref _sent); + + return new { messageId = response.MessageId, sequenceNumber = response.SequenceNumber }; + } + + public async Task TestConnection() + { + var steps = new List(); + try + { + using var probe = CreateClient(_options); + + var listed = await probe.ListQueuesAsync(new ListQueuesRequest { MaxResults = 1 }); + steps.Add(new { step = "credentials", ok = true, detail = _options.Region }); + + foreach (var endpoint in _endpoints) + { + try + { + var attributes = await probe.GetQueueAttributesAsync(endpoint, + new List { "ApproximateNumberOfMessages", "VisibilityTimeout" }); + + steps.Add(new + { + step = $"queue:{Short(endpoint)}", + ok = true, + detail = $"{attributes.ApproximateNumberOfMessages} message(s), " + + $"visibility {attributes.VisibilityTimeout}s" + }); + + // A visibility timeout shorter than Bitween's persist time means duplicate + // processing, so say so before it happens in production. + if (attributes.VisibilityTimeout < 30) + steps.Add(new + { + step = $"queue:{Short(endpoint)}:warning", + ok = true, + detail = $"Visibility timeout is {attributes.VisibilityTimeout}s. " + + "Anything under ~30s risks redelivery while Bitween is still persisting." + }); + } + catch (Exception ex) + { + steps.Add(new { step = $"queue:{Short(endpoint)}", ok = false, detail = ex.Message }); + return new { ok = false, steps }; + } + } + + return new { ok = true, steps }; + } + catch (Exception ex) + { + steps.Add(new { step = "failed", ok = false, detail = ex.Message }); + return new { ok = false, steps }; + } + } + + /// Lists the queues these credentials can see, for the "pick a queue" step in the UI. + public async Task Discover() + { + try + { + var listed = await _sqs.ListQueuesAsync(new ListQueuesRequest { MaxResults = 100 }); + return new + { + region = _options.Region, + queues = listed.QueueUrls.Select(url => new { url, name = Short(url) }), + consuming = _endpoints + }; + } + catch (Exception ex) + { + return new { error = ex.Message, hint = "ListQueues needs sqs:ListQueues on the principal." }; + } + } + + public Task GetStats() => Task.FromResult(new + { + received = _received, deleted = _deleted, returned = _returned, failed = _failed, sent = _sent, + endpoints = _endpoints, visibilityTimeoutSeconds = _options.VisibilityTimeoutSeconds + }); + + public class PublishRequest + { + /// The queue URL. + public string Endpoint { get; set; } + public string Body { get; set; } + public string GroupId { get; set; } + public string DeduplicationId { get; set; } + } +} diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs new file mode 100644 index 00000000..5f2fb93d --- /dev/null +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs @@ -0,0 +1,37 @@ +namespace SW.Bitween.Adapters.Bus.Sqs; + +public class SqsOptions +{ + public string Region { get; set; } = "eu-west-1"; + + /// + /// Leave both blank to use the ambient credential chain — instance profile, IRSA, or the + /// environment. That is the right answer on AWS; explicit keys are for everything else. + /// + public string AccessKeyId { get; set; } + public string SecretAccessKey { get; set; } + + /// Override for LocalStack or ElasticMQ in development. + public string ServiceUrl { get; set; } + + /// Comma-separated queue URLs, supplied by the gateways bound to this data source. + public string Endpoints { get; set; } + + /// Long polling. 20 is the maximum and the only sensible value — 0 burns money and API calls. + public int WaitTimeSeconds { get; set; } = 20; + + /// Max 10 per receive; that is an SQS limit, not a choice. + public int MaxMessagesPerReceive { get; set; } = 10; + + /// + /// How long a received message stays invisible to other consumers. It must exceed the time + /// Bitween takes to persist, or the message is redelivered while still being handled. + /// + public int VisibilityTimeoutSeconds { get; set; } = 60; + + /// + /// SP-API wraps its notifications in an envelope. On, the adapter forwards only the payload + /// and promotes notificationType and the SP-API metadata into headers. + /// + public bool UnwrapSellingPartnerNotification { get; set; } +} diff --git a/SW.Bitween.Api/Data/BitweenDbContext.cs b/SW.Bitween.Api/Data/BitweenDbContext.cs index a95975e2..62cc14f0 100644 --- a/SW.Bitween.Api/Data/BitweenDbContext.cs +++ b/SW.Bitween.Api/Data/BitweenDbContext.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Newtonsoft.Json; using SW.Bitween.Domain.Accounts; +using SW.Bitween.Domain.DataSources; using SW.Bitween.Domain.Gateway; using SW.Bitween.JsonConverters; @@ -130,6 +131,28 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Restrict); bg.HasMany(p => p.Routes).WithOne(p => p.BusGateway).HasForeignKey(p => p.BusGatewayId) .OnDelete(DeleteBehavior.Restrict); + + // Nullable on purpose: null keeps meaning "the internal bus", so no existing row + // changes behaviour and the migration is additive only. + bg.HasOne(p => p.DataSource).WithMany().HasForeignKey(p => p.DataSourceId) + .IsRequired(false).OnDelete(DeleteBehavior.Restrict); + bg.Property(p => p.Endpoint).HasMaxLength(500).IsUnicode(false); + bg.Property(p => p.EndpointProperties).StoreAsJson(); + }); + + modelBuilder.Entity(ds => + { + ds.ToTable("DataSources"); + ds.HasKey(i => i.Id); + ds.Property(i => i.Id).ValueGeneratedOnAdd(); + ds.Property(p => p.Name).IsRequired().HasMaxLength(200); + ds.Property(p => p.AdapterId).IsRequired().HasMaxLength(200).IsUnicode(false); + ds.Property(p => p.Kind).HasConversion(); + ds.Property(p => p.Properties).StoreAsJson(); + ds.Property(p => p.SecretProperties).StoreAsJson(); + ds.Property(p => p.LastKnownState).HasMaxLength(100).IsUnicode(false); + ds.Property(p => p.OwnedByNode).HasMaxLength(200).IsUnicode(false); + ds.HasIndex(p => p.Name).IsUnique(); }); modelBuilder.Entity(bgr => diff --git a/SW.Bitween.Api/Domain/DataSources/DataSource.cs b/SW.Bitween.Api/Domain/DataSources/DataSource.cs new file mode 100644 index 00000000..1c73784c --- /dev/null +++ b/SW.Bitween.Api/Domain/DataSources/DataSource.cs @@ -0,0 +1,80 @@ +using SW.PrimitiveTypes; +using System; +using System.Collections.Generic; + +namespace SW.Bitween.Domain.DataSources; + +/// +/// How to reach an external system, separate from what Bitween does with it. +/// +/// A with no DataSourceId still means the INTERNAL bus, so +/// every gateway that exists today keeps working untouched. Setting one moves that gateway onto an +/// external broker, served by a resident serverless adapter. +/// +/// The split matters: this holds the connection — endpoint, credentials, health — and the gateway +/// holds the meaning — which Document, which routes, which filters. One data source can serve many +/// gateways, exactly as one RabbitMQ connection serves many queues. +/// +public class DataSource : BaseEntity, IAudited +{ + public string Name { get; set; } + + /// + /// The adapter that knows this protocol, e.g. bitween.bus.rabbitmq or + /// bitween.bus.sqs. Resolved and installed from cloud storage like any other adapter. + /// + public string AdapterId { get; set; } + + /// Free text for the UI; the adapter is the authority on what it actually speaks. + public DataSourceKind Kind { get; set; } = DataSourceKind.Broker; + + /// + /// Connection settings handed to the adapter as startup values. Deliberately untyped: a + /// provider must not be constrained to the subset of a broker's model that Bitween happens to + /// have modelled. Secrets are protected at rest — see . + /// + public Dictionary Properties { get; set; } = new(); + + /// + /// Names within whose values are encrypted at rest and never returned + /// by the API in clear. + /// + public List SecretProperties { get; set; } = new(); + + /// Stops the adapter without deleting the configuration, mirroring BusGateway.Inactive. + public bool Inactive { get; set; } + + // ---------------------------------------------------------------- health + + /// Last state the adapter reported on its heartbeat: Connected, Idle, Disconnected... + public string LastKnownState { get; set; } + + public DateTime? LastHeartbeatOn { get; set; } + public string LastException { get; set; } + public int ConsecutiveFailures { get; set; } + + /// + /// Which node currently owns this connection. A broker connection is exclusive, so exactly one + /// node may hold it; this is what the placement layer writes. + /// + public string OwnedByNode { get; set; } + + public DateTime CreatedOn { get; set; } + public string CreatedBy { get; set; } + public DateTime? ModifiedOn { get; set; } + public string ModifiedBy { get; set; } +} + +/// +/// Broker today. The others are declared now because the adapter contract is identical for them — +/// a relational or object-store source differs in whether it pushes or is polled, not in how it is +/// configured, supervised or observed. +/// +public enum DataSourceKind +{ + Broker = 0, + Relational = 1, + Document = 2, + ObjectStore = 3, + Http = 4 +} diff --git a/SW.Bitween.Api/Domain/Gateway/BusGateway.cs b/SW.Bitween.Api/Domain/Gateway/BusGateway.cs index 0e61eaa5..22452677 100644 --- a/SW.Bitween.Api/Domain/Gateway/BusGateway.cs +++ b/SW.Bitween.Api/Domain/Gateway/BusGateway.cs @@ -14,6 +14,28 @@ public class BusGateway : BaseEntity, IAudited /// message. See . /// public bool Inactive { get; set; } + + /// + /// Null means the INTERNAL bus — the only behaviour that existed before, and still the + /// default, so every gateway already in a database keeps working with no migration of data. + /// Set it and this gateway is fed by an external broker through a resident adapter instead. + /// + public int? DataSourceId { get; set; } + public DataSources.DataSource DataSource { get; set; } + + /// + /// Which queue, topic or subscription on that data source feeds this gateway. Meaningless for + /// the internal bus, where the Document's own BusMessageTypeName does the routing. + /// + public string Endpoint { get; set; } + + /// + /// Per-gateway overrides handed to the adapter alongside the data source's own properties — + /// prefetch, consumer group, visibility timeout. Connection settings belong on the DataSource; + /// these are about this one subscription to it. + /// + public Dictionary EndpointProperties { get; set; } = new(); + public ICollection Routes { get; set; } public DateTime CreatedOn { get; set; } public string CreatedBy { get; set; } diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj index da390edb..791a64fd 100644 --- a/SW.Bitween.Api/SW.Bitween.Api.csproj +++ b/SW.Bitween.Api/SW.Bitween.Api.csproj @@ -5,6 +5,12 @@ SW.Bitween + + + + + diff --git a/SW.Bitween.Api/Services/BitweenOptions.cs b/SW.Bitween.Api/Services/BitweenOptions.cs index d9d426a3..c4021397 100644 --- a/SW.Bitween.Api/Services/BitweenOptions.cs +++ b/SW.Bitween.Api/Services/BitweenOptions.cs @@ -15,6 +15,8 @@ public BitweenOptions() DatabaseType = "MySql"; AdminDatabaseName = "defaultdb"; ServerlessCommandTimeout = 300; + BusProvidersEnabled = false; + BusProviderMaxInFlight = 16; ApiCallSubscriptionResponseAcceptedStatusCode = 202; StorageProvider = "S3"; JwtExpiryMinutes = 60; @@ -36,6 +38,15 @@ public BitweenOptions() public string AdminCredentials { get; set; } public string DocumentPrefix { get; set; } public int ServerlessCommandTimeout { get; set; } + + /// + /// Runs BusGateways whose DataSourceId is set, through resident adapters. Off by default + /// because a broker connection is exclusive and node placement is not implemented yet. + /// + public bool BusProvidersEnabled { get; set; } + + /// Unacknowledged messages one bus adapter may have in flight with the host. + public int BusProviderMaxInFlight { get; set; } public bool AreXChangeFilesPrivate { get; set; } = false; public int? ApiCallSubscriptionResponseAcceptedStatusCode { get; set; } diff --git a/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs b/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs new file mode 100644 index 00000000..fc23a32b --- /dev/null +++ b/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain.Gateway; +using SW.EfCoreExtensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using System; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace SW.Bitween.Services.DataSources; + +/// +/// Where a message from an external broker enters Bitween. +/// +/// The adapter owns the connection and hands the payload over; everything after that is the path +/// an internally-bussed message already takes — , +/// which persists the Xchange, writes the payload to cloud storage and lets the filter decide +/// which subscriptions run. Ingress from a broker and ingress from the API are the same thing +/// past this point, deliberately. +/// +/// The adapter does NOT acknowledge its broker until this returns Accepted. That ordering is the +/// whole contract: persist, then ack. A crash in between means redelivery, which is why every +/// event carries a dedupe key. +/// +public class BusProviderEventSink : IAdapterEventSink +{ + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public BusProviderEventSink(IServiceProvider serviceProvider, ILogger logger) + { + _serviceProvider = serviceProvider; + _logger = logger; + } + + public async Task OnEventAsync(InboundEvent inboundEvent, CancellationToken cancellationToken) + { + // A resident adapter is a singleton and outlives any request, so ingest gets its own scope + // per message rather than borrowing one. + using var scope = _serviceProvider.CreateScope(); + + var dbContext = scope.ServiceProvider.GetRequiredService(); + var xchangeService = scope.ServiceProvider.GetRequiredService(); + + if (!int.TryParse(inboundEvent.InstanceKey, out var dataSourceId)) + return EventOutcome.Rejected($"'{inboundEvent.InstanceKey}' is not a data source id."); + + // Which gateway on this data source owns this endpoint. Endpoint is what the adapter was + // told to consume — a queue, a topic, an SQS URL. + var gateway = await dbContext.Set() + .Where(g => g.DataSourceId == dataSourceId && !g.Inactive) + .Where(g => g.Endpoint == inboundEvent.Endpoint || g.Endpoint == null) + .OrderByDescending(g => g.Endpoint) // an exact endpoint match beats the catch-all + .FirstOrDefaultAsync(cancellationToken); + + if (gateway == null) + { + // Not an error: the adapter is consuming something no gateway claims. Rejecting would + // requeue it forever, so accept and drop with a warning instead. + _logger.LogWarning( + "No active bus gateway on data source {DataSourceId} claims endpoint '{Endpoint}'. Discarding.", + dataSourceId, inboundEvent.Endpoint); + return EventOutcome.Ok("unclaimed"); + } + + try + { + var payload = Encoding.UTF8.GetString(inboundEvent.Payload ?? Array.Empty()); + var file = new XchangeFile(payload); + + // Same entry point the internal bus uses. Filtering, mapping, handling, auto-retry and + // the audit trail all follow from here unchanged. + await xchangeService.SubmitFilterXchange( + gateway.DocumentId, + file, + references: string.IsNullOrEmpty(inboundEvent.DedupeKey) + ? null + : new[] { inboundEvent.DedupeKey }, + correlationId: inboundEvent.Traceparent); + + return EventOutcome.Ok(inboundEvent.DedupeKey); + } + catch (Exception ex) + { + // Rejecting is the right answer: the adapter nacks, the broker redelivers, and nothing + // is silently lost because Bitween happened to be unhealthy for a moment. + _logger.LogError(ex, "Failed to ingest a message from data source {DataSourceId} endpoint {Endpoint}.", + dataSourceId, inboundEvent.Endpoint); + return EventOutcome.Rejected(ex.Message); + } + } +} diff --git a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs new file mode 100644 index 00000000..bb5c5c13 --- /dev/null +++ b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs @@ -0,0 +1,200 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Serverless.Resident; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace SW.Bitween.Services.DataSources; + +/// +/// Reconciles running adapters against the data sources this node should be serving. +/// +/// Desired state is the set of active rows; actual state is what the +/// resident host is running. The loop starts what is missing, stops what is no longer wanted, and +/// restarts what has changed — the same shape as the Quartz schedule reconciliation that already +/// exists for subscriptions. +/// +/// PLACEMENT IS NOT DONE HERE YET. A broker connection is exclusive, so exactly one node may hold +/// it; today every node would try. Until leader election lands, run this on a single instance or +/// leave off. The DataSource.OwnedByNode column +/// exists for that election to write into. +/// +public class BusProviderSupervisor : BackgroundService +{ + private static readonly TimeSpan ReconcileInterval = TimeSpan.FromSeconds(30); + + private readonly IServiceProvider _serviceProvider; + private readonly IResidentAdapterHost _adapters; + private readonly ILogger _logger; + + // What we last started, and the configuration fingerprint it was started with. + private readonly Dictionary _running = new(); + + public BusProviderSupervisor(IServiceProvider serviceProvider, IResidentAdapterHost adapters, + ILogger logger) + { + _serviceProvider = serviceProvider; + _adapters = adapters; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await ReconcileAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + // Never let one bad reconcile end the loop; the next pass retries everything. + _logger.LogError(ex, "Bus provider reconciliation failed."); + } + + try { await Task.Delay(ReconcileInterval, stoppingToken); } + catch (OperationCanceledException) { return; } + } + } + + private async Task ReconcileAsync(CancellationToken cancellationToken) + { + using var scope = _serviceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var desired = await dbContext.Set() + .Where(d => !d.Inactive) + .AsNoTracking() + .ToListAsync(cancellationToken); + + // Endpoint configuration lives on the gateways, so an adapter is told what to consume by + // the gateways pointing at it rather than by the data source alone. + var endpoints = await dbContext.Set() + .Where(g => g.DataSourceId != null && !g.Inactive) + .AsNoTracking() + .ToListAsync(cancellationToken); + + foreach (var dataSource in desired) + { + var startupValues = BuildStartupValues(dataSource, endpoints); + var fingerprint = Fingerprint(dataSource, startupValues); + + if (_running.TryGetValue(dataSource.Id, out var current)) + { + if (current == fingerprint) continue; + + _logger.LogInformation("Data source {Name} changed; restarting its adapter.", dataSource.Name); + await _adapters.StopAsync(dataSource.AdapterId, dataSource.Id.ToString(), + drain: true, cancellationToken); + _running.Remove(dataSource.Id); + } + + try + { + await _adapters.StartExclusiveAsync(new AdapterSpec + { + AdapterId = dataSource.AdapterId, + // The instance key IS the data source id, which is how the sink knows which + // gateway an inbound message belongs to. + InstanceKey = dataSource.Id.ToString(), + StartupValues = startupValues + }, cancellationToken); + + _running[dataSource.Id] = fingerprint; + _logger.LogInformation("Data source {Name} running on adapter {AdapterId}.", + dataSource.Name, dataSource.AdapterId); + } + catch (Exception ex) + { + // One unreachable broker must not stop the others from starting. + _logger.LogError(ex, "Could not start adapter {AdapterId} for data source {Name}.", + dataSource.AdapterId, dataSource.Name); + } + } + + // Anything running that is no longer desired. + var wanted = desired.Select(d => d.Id).ToHashSet(); + foreach (var id in _running.Keys.Where(k => !wanted.Contains(k)).ToList()) + { + var adapterId = desired.FirstOrDefault(d => d.Id == id)?.AdapterId; + if (adapterId != null) + await _adapters.StopAsync(adapterId, id.ToString(), drain: true, cancellationToken); + _running.Remove(id); + } + + await WriteBackHealthAsync(dbContext, cancellationToken); + } + + /// + /// Connection settings from the data source, plus the endpoints its gateways want consumed. + /// The adapter decides what to do with them — Bitween does not model any broker's topology. + /// + private static Dictionary BuildStartupValues(DataSource dataSource, + IEnumerable gateways) + { + var values = new Dictionary(dataSource.Properties ?? new Dictionary(), + StringComparer.OrdinalIgnoreCase); + + var mine = gateways.Where(g => g.DataSourceId == dataSource.Id).ToList(); + + var wanted = mine.Where(g => !string.IsNullOrWhiteSpace(g.Endpoint)) + .Select(g => g.Endpoint) + .Distinct() + .ToList(); + + if (wanted.Count > 0) values["Endpoints"] = string.Join(",", wanted); + + // Per-gateway overrides, namespaced so they cannot collide with connection settings. + foreach (var gateway in mine) + foreach (var kv in gateway.EndpointProperties ?? new()) + values[$"Endpoint:{gateway.Endpoint}:{kv.Key}"] = kv.Value; + + return values; + } + + private static string Fingerprint(DataSource dataSource, Dictionary startupValues) => + dataSource.AdapterId + "|" + + string.Join(";", startupValues.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}")); + + /// + /// Health from the heartbeat, written back so the UI and the notifiers can see a broker that + /// has gone away without anyone tailing logs. + /// + private async Task WriteBackHealthAsync(BitweenDbContext dbContext, CancellationToken cancellationToken) + { + var health = _adapters.Describe().ToDictionary(h => h.InstanceKey); + if (health.Count == 0) return; + + var ids = health.Keys.Select(k => int.TryParse(k, out var id) ? id : 0).Where(i => i > 0).ToList(); + var rows = await dbContext.Set() + .Where(d => ids.Contains(d.Id)) + .ToListAsync(cancellationToken); + + var changed = false; + + foreach (var row in rows) + { + if (!health.TryGetValue(row.Id.ToString(), out var instance)) continue; + + row.LastKnownState = instance.ReportedState ?? instance.State.ToString(); + row.LastHeartbeatOn = instance.LastHeartbeatOn?.UtcDateTime; + row.LastException = instance.LastError; + row.ConsecutiveFailures = instance.RestartCount; + row.OwnedByNode = Environment.MachineName; + changed = true; + } + + if (changed) await dbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index 7415050f..9f0e976d 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -27,7 +27,9 @@ - + + diff --git a/SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.Designer.cs b/SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.Designer.cs new file mode 100644 index 00000000..0bda7605 --- /dev/null +++ b/SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.Designer.cs @@ -0,0 +1,2553 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SW.Bitween.Model; +using SW.Bitween.PgSql; + +#nullable disable + +namespace SW.Bitween.PgSql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906011231_ExternalBusDataSources")] + partial class ExternalBusDataSources + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("infolink") + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "hstore"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Deleted") + .HasColumnType("boolean") + .HasColumnName("deleted"); + + b.Property("Disabled") + .HasColumnType("boolean") + .HasColumnName("disabled"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("display_name"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("email"); + + b.Property("EmailProvider") + .HasColumnType("smallint") + .HasColumnName("email_provider"); + + b.Property("FailedLoginCount") + .HasColumnType("integer") + .HasColumnName("failed_login_count"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone") + .HasColumnName("lockout_end"); + + b.Property("LoginMethods") + .HasColumnType("smallint") + .HasColumnName("login_methods"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("character varying(500)") + .HasColumnName("password"); + + b.Property("Role") + .HasColumnType("integer") + .HasColumnName("role"); + + b.HasKey("Id") + .HasName("pk_accounts"); + + b.HasIndex("Email") + .IsUnique() + .HasDatabaseName("ix_accounts_email"); + + b.ToTable("Accounts", "infolink"); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("integer") + .HasColumnName("account_id"); + + b.Property("RoleId") + .HasColumnType("integer") + .HasColumnName("role_id"); + + b.HasKey("AccountId", "RoleId") + .HasName("pk_account_roles"); + + b.HasIndex("RoleId") + .HasDatabaseName("ix_account_roles_role_id"); + + b.ToTable("AccountRoles", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("integer") + .HasColumnName("account_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("LoginMethod") + .HasColumnType("smallint") + .HasColumnName("login_method"); + + b.HasKey("Id") + .HasName("pk_refresh_tokens"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_refresh_tokens_account_id"); + + b.ToTable("RefreshTokens", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("description"); + + b.Property("IsSystem") + .HasColumnType("boolean") + .HasColumnName("is_system"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("Permissions") + .HasColumnType("text") + .HasColumnName("permissions"); + + b.HasKey("Id") + .HasName("pk_roles"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_roles_name"); + + b.ToTable("Roles", "infolink"); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdapterId") + .HasColumnType("text") + .HasColumnName("adapter_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Kind") + .HasColumnType("integer") + .HasColumnName("kind"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("LastHeartbeatOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_heartbeat_on"); + + b.Property("LastKnownState") + .HasColumnType("text") + .HasColumnName("last_known_state"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("OwnedByNode") + .HasColumnType("text") + .HasColumnName("owned_by_node"); + + b.Property>("Properties") + .HasColumnType("hstore") + .HasColumnName("properties"); + + b.PrimitiveCollection>("SecretProperties") + .HasColumnType("text[]") + .HasColumnName("secret_properties"); + + b.HasKey("Id") + .HasName("pk_data_source"); + + b.ToTable("data_source", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("On") + .HasColumnType("timestamp with time zone") + .HasColumnName("on"); + + b.HasKey("Id") + .HasName("pk_delayed_retry"); + + b.HasIndex("On") + .HasDatabaseName("ix_delayed_retry_on"); + + b.ToTable("delayed_retry", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusEnabled") + .HasColumnType("boolean") + .HasColumnName("bus_enabled"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("bus_message_type_name"); + + b.Property("Code") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("code"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("boolean") + .HasColumnName("disregards_unfiltered_messages"); + + b.Property("DocumentFormat") + .HasColumnType("integer") + .HasColumnName("document_format"); + + b.Property("DuplicateInterval") + .HasColumnType("integer") + .HasColumnName("duplicate_interval"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("PromotedProperties") + .HasColumnType("jsonb") + .HasColumnName("promoted_properties"); + + b.HasKey("Id") + .HasName("pk_document"); + + b.HasIndex("BusMessageTypeName") + .IsUnique() + .HasDatabaseName("ix_document_bus_message_type_name"); + + b.HasIndex("Code") + .IsUnique() + .HasDatabaseName("ix_document_code"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_document_name"); + + b.ToTable("document", "infolink"); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("Code") + .HasColumnType("integer") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("StateAfter") + .HasColumnType("text") + .HasColumnName("state_after"); + + b.Property("StateBefore") + .HasColumnType("text") + .HasColumnName("state_before"); + + b.HasKey("Id") + .HasName("pk_document_trail"); + + b.HasIndex("CreatedOn") + .HasDatabaseName("ix_document_trail_created_on"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_document_trail_document_id"); + + b.ToTable("document_trail", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("url_name"); + + b.HasKey("Id") + .HasName("pk_api_gateway"); + + b.HasIndex("UrlName") + .IsUnique() + .HasDatabaseName("ix_api_gateway_url_name"); + + b.ToTable("api_gateway", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("integer") + .HasColumnName("api_gateway_id"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId") + .HasName("pk_api_gateway_partner"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_api_gateway_partner_partner_id"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_api_gateway_partner_subscription_id"); + + b.ToTable("api_gateway_partner", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DataSourceId") + .HasColumnType("integer") + .HasColumnName("data_source_id"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property>("EndpointProperties") + .HasColumnType("hstore") + .HasColumnName("endpoint_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_bus_gateway"); + + b.HasIndex("DataSourceId") + .HasDatabaseName("ix_bus_gateway_data_source_id"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_bus_gateway_document_id"); + + b.ToTable("bus_gateway", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("integer") + .HasColumnName("bus_gateway_id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("MatchExpression") + .HasColumnType("text") + .HasColumnName("match_expression"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_bus_gateway_route"); + + b.HasIndex("BusGatewayId") + .HasDatabaseName("ix_bus_gateway_route_bus_gateway_id"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_bus_gateway_route_partner_id"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_bus_gateway_route_subscription_id"); + + b.ToTable("bus_gateway_route", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property>("Values") + .HasColumnType("jsonb") + .HasColumnName("values"); + + b.HasKey("Id") + .HasName("pk_global_adapter_values_set"); + + b.ToTable("global_adapter_values_set", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property("HandlerProperties") + .HasColumnType("text") + .HasColumnName("handler_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("RunOnBadResult") + .HasColumnType("boolean") + .HasColumnName("run_on_bad_result"); + + b.Property("RunOnFailedResult") + .HasColumnType("boolean") + .HasColumnName("run_on_failed_result"); + + b.PrimitiveCollection("RunOnSubscriptions") + .HasColumnType("integer[]") + .HasColumnName("run_on_subscriptions"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("boolean") + .HasColumnName("run_on_successful_result"); + + b.HasKey("Id") + .HasName("pk_notifier"); + + b.ToTable("notifier", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("boolean") + .HasColumnName("bad_data"); + + b.Property("Data") + .HasColumnType("text") + .HasColumnName("data"); + + b.Property("FileName") + .HasColumnType("text") + .HasColumnName("file_name"); + + b.PrimitiveCollection("References") + .HasColumnType("text[]") + .HasColumnName("references"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_on_hold_xchange"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_on_hold_xchange_subscription_id"); + + b.ToTable("on_hold_xchange", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property>("AdapterProperties") + .HasColumnType("jsonb") + .HasColumnName("adapter_properties"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_partner"); + + b.ToTable("partner", "infolink"); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasColumnType("text") + .HasColumnName("error_message"); + + b.PrimitiveCollection("ExchangeIds") + .HasColumnType("text[]") + .HasColumnName("exchange_ids"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("Outcome") + .HasColumnType("integer") + .HasColumnName("outcome"); + + b.Property("StartedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_on"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_receive_attempt"); + + b.HasIndex("SubscriptionId", "StartedOn") + .HasDatabaseName("ix_receive_attempt_subscription_id_started_on"); + + b.ToTable("receive_attempt", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("GroupId") + .HasColumnType("uuid") + .HasColumnName("group_id"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("alert_handler_id"); + + b.Property("AlertHandlerProperties") + .HasColumnType("text") + .HasColumnName("alert_handler_properties"); + + b.Property("AlertMode") + .HasColumnType("smallint") + .HasColumnName("alert_mode"); + + b.HasKey("SubscriptionId", "GroupId") + .HasName("pk_retry_alert_override"); + + b.ToTable("retry_alert_override", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("GroupId") + .HasColumnType("uuid") + .HasColumnName("group_id"); + + b.Property("AttemptsUsed") + .HasColumnType("integer") + .HasColumnName("attempts_used"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("exhausted_notified_on"); + + b.Property("LastAttemptOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempt_on"); + + b.HasKey("SubscriptionId", "GroupId") + .HasName("pk_retry_group_usage"); + + b.ToTable("retry_group_usage", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("alert_handler_id"); + + b.Property("AlertHandlerProperties") + .HasColumnType("text") + .HasColumnName("alert_handler_properties"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Groups") + .HasColumnType("text") + .HasColumnName("groups"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_retry_policy"); + + b.ToTable("retry_policy", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Value") + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Id") + .HasName("pk_settings"); + + b.ToTable("Settings", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("aggregate_on"); + + b.Property("AggregationForId") + .HasColumnType("integer") + .HasColumnName("aggregation_for_id"); + + b.Property("AggregationTarget") + .HasColumnType("smallint") + .HasColumnName("aggregation_target"); + + b.Property("CategoryId") + .HasColumnType("integer") + .HasColumnName("category_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CustomRetryPolicy") + .HasColumnType("text") + .HasColumnName("custom_retry_policy"); + + b.Property>("DocumentFilter") + .HasColumnType("jsonb") + .HasColumnName("document_filter"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("HandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property>("HandlerProperties") + .HasColumnType("jsonb") + .HasColumnName("handler_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("IsRunning") + .HasColumnType("boolean") + .HasColumnName("is_running"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("MapperId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("mapper_id"); + + b.Property>("MapperProperties") + .HasColumnType("jsonb") + .HasColumnName("mapper_properties"); + + b.Property("MatchExpression") + .HasColumnType("text") + .HasColumnName("match_expression"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("PausedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("paused_on"); + + b.Property("ReceiveOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("receive_on"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("receiver_id"); + + b.Property>("ReceiverProperties") + .HasColumnType("jsonb") + .HasColumnName("receiver_properties"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("response_message_type_name"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("integer") + .HasColumnName("response_subscription_id"); + + b.Property("RetryPolicyId") + .HasColumnType("integer") + .HasColumnName("retry_policy_id"); + + b.Property("Temporary") + .HasColumnType("boolean") + .HasColumnName("temporary"); + + b.Property("Type") + .HasColumnType("smallint") + .HasColumnName("type"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("validator_id"); + + b.Property>("ValidatorProperties") + .HasColumnType("jsonb") + .HasColumnName("validator_properties"); + + b.Property("WorkGroupId") + .HasColumnType("integer") + .HasColumnName("work_group_id"); + + b.HasKey("Id") + .HasName("pk_subscription"); + + b.HasIndex("AggregationForId") + .HasDatabaseName("ix_subscription_aggregation_for_id"); + + b.HasIndex("CategoryId") + .HasDatabaseName("ix_subscription_category_id"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_subscription_document_id"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_subscription_partner_id"); + + b.HasIndex("ResponseSubscriptionId") + .HasDatabaseName("ix_subscription_response_subscription_id"); + + b.HasIndex("RetryPolicyId") + .HasDatabaseName("ix_subscription_retry_policy_id"); + + b.HasIndex("WorkGroupId") + .HasDatabaseName("ix_subscription_work_group_id"); + + b.ToTable("subscription", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("text") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.HasKey("Id") + .HasName("pk_subscription_category"); + + b.HasIndex("Code") + .IsUnique() + .HasDatabaseName("ix_subscription_category_code"); + + b.ToTable("subscription_category", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("Code") + .HasColumnType("integer") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("StateAfter") + .HasColumnType("text") + .HasColumnName("state_after"); + + b.Property("StateBefore") + .HasColumnType("text") + .HasColumnName("state_before"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_subscription_trail"); + + b.HasIndex("CreatedOn") + .HasDatabaseName("ix_subscription_trail_created_on"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_subscription_trail_subscription_id"); + + b.ToTable("subscription_trail", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("character varying(100)") + .HasColumnName("bus_message_name"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Options") + .HasColumnType("jsonb") + .HasColumnName("options"); + + b.HasKey("Id") + .HasName("pk_work_group"); + + b.ToTable("work_group", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("CorrelationId") + .HasColumnType("text") + .HasColumnName("correlation_id"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("HandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property>("HandlerProperties") + .HasColumnType("jsonb") + .HasColumnName("handler_properties"); + + b.Property("InputContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("input_content_type"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("input_hash"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("input_name"); + + b.Property("InputSize") + .HasColumnType("integer") + .HasColumnName("input_size"); + + b.Property("ManualRetry") + .HasColumnType("boolean") + .HasColumnName("manual_retry"); + + b.Property("MapperId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("mapper_id"); + + b.Property>("MapperProperties") + .HasColumnType("jsonb") + .HasColumnName("mapper_properties"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.PrimitiveCollection("References") + .HasColumnType("text[]") + .HasColumnName("references"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("response_message_type_name"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("integer") + .HasColumnName("response_subscription_id"); + + b.Property("RetryFor") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("retry_for"); + + b.Property("StartedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_on"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_xchange"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_xchange_document_id"); + + b.HasIndex("InputHash") + .HasDatabaseName("ix_xchange_input_hash"); + + b.HasIndex("RetryFor") + .HasDatabaseName("ix_xchange_retry_for"); + + b.HasIndex("StartedOn") + .HasDatabaseName("ix_xchange_started_on"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_xchange_subscription_id"); + + b.ToTable("xchange", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AggregatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("aggregated_on"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("aggregation_xchange_id"); + + b.HasKey("Id") + .HasName("pk_xchange_aggregation"); + + b.HasIndex("AggregationXchangeId") + .HasDatabaseName("ix_xchange_aggregation_aggregation_xchange_id"); + + b.ToTable("xchange_aggregation", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("DeliveredOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("delivered_on"); + + b.HasKey("Id") + .HasName("pk_xchange_delivery"); + + b.HasIndex("DeliveredOn") + .HasDatabaseName("ix_xchange_delivery_delivered_on"); + + b.ToTable("xchange_delivery", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("text") + .HasColumnName("exception"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("NotifierId") + .HasColumnType("integer") + .HasColumnName("notifier_id"); + + b.Property("NotifierName") + .HasColumnType("text") + .HasColumnName("notifier_name"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("character varying(50)") + .HasColumnName("xchange_id"); + + b.HasKey("Id") + .HasName("pk_xchange_notification"); + + b.ToTable("xchange_notification", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.PrimitiveCollection("Hits") + .HasColumnType("integer[]") + .HasColumnName("hits"); + + b.Property>("Properties") + .HasColumnType("jsonb") + .HasColumnName("properties"); + + b.Property("PropertiesRaw") + .HasColumnType("text") + .HasColumnName("properties_raw"); + + b.HasKey("Id") + .HasName("pk_xchange_promoted_properties"); + + b.HasIndex("PropertiesRaw") + .HasDatabaseName("ix_xchange_promoted_properties_properties_raw"); + + b.ToTable("xchange_promoted_properties", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AttemptNumber") + .HasColumnType("integer") + .HasColumnName("attempt_number"); + + b.Property("Exception") + .HasColumnType("text") + .HasColumnName("exception"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("OutputBad") + .HasColumnType("boolean") + .HasColumnName("output_bad"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("output_content_type"); + + b.Property("OutputHash") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("output_hash"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("output_name"); + + b.Property("OutputSize") + .HasColumnType("integer") + .HasColumnName("output_size"); + + b.Property("ResponseBad") + .HasColumnType("boolean") + .HasColumnName("response_bad"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("response_content_type"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("response_hash"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("response_name"); + + b.Property("ResponseSize") + .HasColumnType("integer") + .HasColumnName("response_size"); + + b.Property("ResponseXchangeId") + .HasColumnType("text") + .HasColumnName("response_xchange_id"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("retry_blocked_reason"); + + b.Property("RetryGroupId") + .HasColumnType("uuid") + .HasColumnName("retry_group_id"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.HasKey("Id") + .HasName("pk_xchange_result"); + + b.HasIndex("RetryGroupId") + .HasDatabaseName("ix_xchange_result_retry_group_id"); + + b.ToTable("xchange_result", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("boolean") + .HasColumnName("is_running"); + + b.ToTable("running_result", "infolink"); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("text") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("text") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.HasKey("Id") + .HasName("pk_job_executions"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("bytea") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_blob_triggers"); + + b.ToTable("qrtz_blob_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName") + .HasName("pk_qrtz_calendars"); + + b.ToTable("qrtz_calendars", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_cron_triggers"); + + b.ToTable("qrtz_cron_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("text") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId") + .HasName("pk_qrtz_fired_triggers"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_qrtz_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_qrtz_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_qrtz_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_qrtz_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_nm_gp"); + + b.ToTable("qrtz_fired_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bool") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bool") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup") + .HasName("pk_qrtz_job_details"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("qrtz_job_details", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("text") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName") + .HasName("pk_qrtz_locks"); + + b.ToTable("qrtz_locks", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup") + .HasName("pk_qrtz_paused_trigger_grps"); + + b.ToTable("qrtz_paused_trigger_grps", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName") + .HasName("pk_qrtz_scheduler_state"); + + b.ToTable("qrtz_scheduler_state", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bool") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bool") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("integer") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("integer") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("text") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("text") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("text") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_simprop_triggers"); + + b.ToTable("qrtz_simprop_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_simple_triggers"); + + b.ToTable("qrtz_simple_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("integer") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_triggers"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup") + .HasDatabaseName("ix_qrtz_triggers_sched_name_job_name_job_group"); + + b.ToTable("qrtz_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_roles_accounts_account_id"); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_roles_roles_role_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_refresh_tokens_accounts_account_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_document_trail_document_document_id"); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_api_gateway_api_gateway_id"); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_subscription_subscription_id"); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .HasConstraintName("fk_bus_gateway_data_source_data_source_id"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_document_document_id"); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_route_bus_gateway_bus_gateway_id"); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_bus_gateway_route_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_route_subscription_subscription_id"); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("key"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("name"); + + b1.HasKey("PartnerId", "Id") + .HasName("pk_partner_api_credential"); + + b1.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ix_partner_api_credential_key"); + + b1.ToTable("partner_api_credential", "infolink"); + + b1.WithOwner() + .HasForeignKey("PartnerId") + .HasConstraintName("fk_partner_api_credential_partner_partner_id"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_aggregation_for"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId") + .HasConstraintName("fk_subscription_subscription_category_category_id"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_subscription_document_document_id"); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_response_subscriber"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_subscription_retry_policy_retry_policy_id"); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId") + .HasConstraintName("fk_subscription_work_group_work_group_id"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("boolean") + .HasColumnName("backwards"); + + b1.Property("On") + .HasColumnType("bigint") + .HasColumnName("on"); + + b1.Property("Recurrence") + .HasColumnType("smallint") + .HasColumnName("recurrence"); + + b1.HasKey("SubscriptionId", "Id") + .HasName("pk_subscription_schedule"); + + b1.ToTable("subscription_schedule", "infolink"); + + b1.WithOwner() + .HasForeignKey("SubscriptionId") + .HasConstraintName("fk_subscription_schedule_subscription_subscription_id"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_subscription_trail_subscription_subscription_id"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_xchange_document_document_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_aggregation_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_delivery_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_promoted_properties_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_result_xchange_id"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_blob_triggers_qrtz_triggers_sched_name_trigger_name_tr"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_cron_triggers_qrtz_triggers_sched_name_trigger_name_tr"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_simprop_triggers_qrtz_triggers_sched_name_trigger_name"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_simple_triggers_qrtz_triggers_sched_name_trigger_name_"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_triggers_qrtz_job_details_sched_name_job_name_job_group"); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.cs b/SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.cs new file mode 100644 index 00000000..9a9d9c66 --- /dev/null +++ b/SW.Bitween.PgSql/Migrations/20260906011231_ExternalBusDataSources.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace SW.Bitween.PgSql.Migrations +{ + /// + public partial class ExternalBusDataSources : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterDatabase() + .Annotation("Npgsql:PostgresExtension:hstore", ",,"); + + migrationBuilder.AddColumn( + name: "data_source_id", + schema: "infolink", + table: "bus_gateway", + type: "integer", + nullable: true); + + migrationBuilder.AddColumn( + name: "endpoint", + schema: "infolink", + table: "bus_gateway", + type: "text", + nullable: true); + + migrationBuilder.AddColumn>( + name: "endpoint_properties", + schema: "infolink", + table: "bus_gateway", + type: "hstore", + nullable: true); + + migrationBuilder.CreateTable( + name: "data_source", + schema: "infolink", + columns: table => new + { + id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + name = table.Column(type: "text", nullable: true), + adapter_id = table.Column(type: "text", nullable: true), + kind = table.Column(type: "integer", nullable: false), + properties = table.Column>(type: "hstore", nullable: true), + secret_properties = table.Column>(type: "text[]", nullable: true), + inactive = table.Column(type: "boolean", nullable: false), + last_known_state = table.Column(type: "text", nullable: true), + last_heartbeat_on = table.Column(type: "timestamp with time zone", nullable: true), + last_exception = table.Column(type: "text", nullable: true), + consecutive_failures = table.Column(type: "integer", nullable: false), + owned_by_node = table.Column(type: "text", nullable: true), + created_on = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "text", nullable: true), + modified_on = table.Column(type: "timestamp with time zone", nullable: true), + modified_by = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_data_source", x => x.id); + }); + + migrationBuilder.CreateIndex( + name: "ix_bus_gateway_data_source_id", + schema: "infolink", + table: "bus_gateway", + column: "data_source_id"); + + migrationBuilder.AddForeignKey( + name: "fk_bus_gateway_data_source_data_source_id", + schema: "infolink", + table: "bus_gateway", + column: "data_source_id", + principalSchema: "infolink", + principalTable: "data_source", + principalColumn: "id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_bus_gateway_data_source_data_source_id", + schema: "infolink", + table: "bus_gateway"); + + migrationBuilder.DropTable( + name: "data_source", + schema: "infolink"); + + migrationBuilder.DropIndex( + name: "ix_bus_gateway_data_source_id", + schema: "infolink", + table: "bus_gateway"); + + migrationBuilder.DropColumn( + name: "data_source_id", + schema: "infolink", + table: "bus_gateway"); + + migrationBuilder.DropColumn( + name: "endpoint", + schema: "infolink", + table: "bus_gateway"); + + migrationBuilder.DropColumn( + name: "endpoint_properties", + schema: "infolink", + table: "bus_gateway"); + + migrationBuilder.AlterDatabase() + .OldAnnotation("Npgsql:PostgresExtension:hstore", ",,"); + } + } +} diff --git a/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs b/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs index 28312a03..ace103af 100644 --- a/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs +++ b/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs @@ -23,6 +23,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasAnnotation("ProductVersion", "9.0.19") .HasAnnotation("Relational:MaxIdentifierLength", 63); + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "hstore"); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => @@ -253,6 +254,81 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdapterId") + .HasColumnType("text") + .HasColumnName("adapter_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Kind") + .HasColumnType("integer") + .HasColumnName("kind"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("LastHeartbeatOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_heartbeat_on"); + + b.Property("LastKnownState") + .HasColumnType("text") + .HasColumnName("last_known_state"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("OwnedByNode") + .HasColumnType("text") + .HasColumnName("owned_by_node"); + + b.Property>("Properties") + .HasColumnType("hstore") + .HasColumnName("properties"); + + b.PrimitiveCollection>("SecretProperties") + .HasColumnType("text[]") + .HasColumnName("secret_properties"); + + b.HasKey("Id") + .HasName("pk_data_source"); + + b.ToTable("data_source", "infolink"); + }); + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => { b.Property("Id") @@ -499,10 +575,22 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("timestamp with time zone") .HasColumnName("created_on"); + b.Property("DataSourceId") + .HasColumnType("integer") + .HasColumnName("data_source_id"); + b.Property("DocumentId") .HasColumnType("integer") .HasColumnName("document_id"); + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property>("EndpointProperties") + .HasColumnType("hstore") + .HasColumnName("endpoint_properties"); + b.Property("Inactive") .HasColumnType("boolean") .HasColumnName("inactive"); @@ -524,6 +612,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id") .HasName("pk_bus_gateway"); + b.HasIndex("DataSourceId") + .HasDatabaseName("ix_bus_gateway_data_source_id"); + b.HasIndex("DocumentId") .HasDatabaseName("ix_bus_gateway_document_id"); @@ -2117,12 +2208,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .HasConstraintName("fk_bus_gateway_data_source_data_source_id"); + b.HasOne("SW.Bitween.Domain.Document", null) .WithMany() .HasForeignKey("DocumentId") .OnDelete(DeleteBehavior.Restrict) .IsRequired() .HasConstraintName("fk_bus_gateway_document_document_id"); + + b.Navigation("DataSource"); }); modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => diff --git a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj index ca017b8a..1ef6acb7 100644 --- a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj +++ b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj @@ -5,6 +5,6 @@ SW.Bitween.SampleConfigurableAdapter - + diff --git a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj index cb83ae75..40c9124e 100644 --- a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj +++ b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj index bc31167c..2b87e8f9 100644 --- a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj +++ b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj index a6ba4657..aa6329d9 100644 --- a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj +++ b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj @@ -8,7 +8,7 @@ - + diff --git a/SW.Bitween.Web/SW.Bitween.Web.csproj b/SW.Bitween.Web/SW.Bitween.Web.csproj index 9efbfd9d..e0131c98 100644 --- a/SW.Bitween.Web/SW.Bitween.Web.csproj +++ b/SW.Bitween.Web/SW.Bitween.Web.csproj @@ -48,7 +48,9 @@ - + + diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index df5ecf46..ed31265b 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -29,6 +29,8 @@ using SW.Bitween.Domain; using SW.Bitween.Resources.Accounts; using SW.Bitween.Services; +using SW.Bitween.Services.DataSources; +using SW.Serverless.Resident; using SW.CqApi.AuthOptions; using SW.Logger.Console; using SW.Logger.ElasticSerach; @@ -158,6 +160,19 @@ public void ConfigureServices(IServiceCollection services) configure.CommandTimeout = bitweenOptions.ServerlessCommandTimeout; configure.AdapterRemotePath = bitweenOptions.AdapterPath; }); + + // External bus providers. Off by default: a broker connection is exclusive, and + // placement across nodes is not implemented yet, so every instance would otherwise + // try to hold the same connection. Turn it on only where a single instance owns them. + if (bitweenOptions.BusProvidersEnabled) + { + services.AddResidentAdapters(configure => + { + configure.HeartbeatInterval = TimeSpan.FromSeconds(15); + configure.MaxInFlight = bitweenOptions.BusProviderMaxInFlight; + }); + services.AddHostedService(); + } services.AddScoped(); // Get and validate connection string diff --git a/SW.Bitween.sln b/SW.Bitween.sln index 47c2b2c7..21cd9456 100644 --- a/SW.Bitween.sln +++ b/SW.Bitween.sln @@ -31,6 +31,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.NativeAdapters", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.IntegrationTests", "SW.Bitween.IntegrationTests\SW.Bitween.IntegrationTests.csproj", "{A2B58DFC-A4B0-4BD2-9552-322C6E87DA39}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Bus Providers", "Bus Providers", "{BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Bus.RabbitMq", "SW.Bitween.Adapters.Bus.RabbitMq\SW.Bitween.Adapters.Bus.RabbitMq.csproj", "{5E93D24F-EA1A-4788-B19F-326215B9CD2B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Sdk", "..\SW-Serverless\SW.Serverless.Sdk\SW.Serverless.Sdk.csproj", "{F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Contract", "..\SW-Serverless\SW.Serverless.Contract\SW.Serverless.Contract.csproj", "{A95E858E-25A6-4A5D-B10F-8D4E914A06F4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Bus.Sqs", "SW.Bitween.Adapters.Bus.Sqs\SW.Bitween.Adapters.Bus.Sqs.csproj", "{1125FB09-88E3-405B-80C0-62C73B1AB9F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -185,6 +195,54 @@ Global {A2B58DFC-A4B0-4BD2-9552-322C6E87DA39}.Release|x64.Build.0 = Release|Any CPU {A2B58DFC-A4B0-4BD2-9552-322C6E87DA39}.Release|x86.ActiveCfg = Release|Any CPU {A2B58DFC-A4B0-4BD2-9552-322C6E87DA39}.Release|x86.Build.0 = Release|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Debug|x64.ActiveCfg = Debug|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Debug|x64.Build.0 = Debug|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Debug|x86.ActiveCfg = Debug|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Debug|x86.Build.0 = Debug|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|Any CPU.Build.0 = Release|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x64.ActiveCfg = Release|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x64.Build.0 = Release|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x86.ActiveCfg = Release|Any CPU + {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x86.Build.0 = Release|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x64.ActiveCfg = Debug|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x64.Build.0 = Debug|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x86.ActiveCfg = Debug|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x86.Build.0 = Debug|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|Any CPU.Build.0 = Release|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x64.ActiveCfg = Release|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x64.Build.0 = Release|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x86.ActiveCfg = Release|Any CPU + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x86.Build.0 = Release|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x64.Build.0 = Debug|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x86.ActiveCfg = Debug|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x86.Build.0 = Debug|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|Any CPU.Build.0 = Release|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x64.ActiveCfg = Release|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x64.Build.0 = Release|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x86.ActiveCfg = Release|Any CPU + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x86.Build.0 = Release|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|x64.ActiveCfg = Debug|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|x64.Build.0 = Debug|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|x86.ActiveCfg = Debug|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|x86.Build.0 = Debug|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|Any CPU.Build.0 = Release|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x64.ActiveCfg = Release|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x64.Build.0 = Release|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x86.ActiveCfg = Release|Any CPU + {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -196,6 +254,11 @@ Global {C78BFACC-0794-4164-A4E9-F5B7FBFD3010} = {5F58DD63-8ABF-4148-A594-0D9881F39142} {CDA68A45-ABF9-46D6-ADED-628CC7CD1200} = {5F58DD63-8ABF-4148-A594-0D9881F39142} {1474658D-E225-478E-80D6-D41A0376F88C} = {DCB20324-CBBC-43BB-9529-6F16C4033A5B} + {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} = {5F58DD63-8ABF-4148-A594-0D9881F39142} + {5E93D24F-EA1A-4788-B19F-326215B9CD2B} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} + {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} + {A95E858E-25A6-4A5D-B10F-8D4E914A06F4} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} + {1125FB09-88E3-405B-80C0-62C73B1AB9F8} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {F314D530-ADF8-43EC-A747-26CA23CCC4F7} diff --git a/docs/external-brokers-architecture.md b/docs/external-brokers-architecture.md new file mode 100644 index 00000000..be576a3b --- /dev/null +++ b/docs/external-brokers-architecture.md @@ -0,0 +1,1134 @@ +# External Brokers: Data Sources, Provider Plugins, and Cluster Control + +Status: **design proposal** (not implemented) +Baseline: **`origin/v2`** @ `fa2dcb3` — 46 commits ahead of `releases/r8.0`, and the branch this +feature lands on. Anything below that cites `releases/r8.0` line numbers is marked as such. + +> **v2 changes several premises.** It embeds the UI in the API (`SW.Bitween.Web/ClientApp`, +> a *new* stack — TanStack Query + Tailwind 4 + Headless UI + react-router 8, no Redux and no +> RTK Query, so `Bitween-UI`'s conventions in `CLAUDE.md` do not apply here), adds an `Ops` +> health API, an RBAC permission system, and — most relevant — a **runtime settings subsystem +> with a descriptor catalog and a secret protector** that this design should reuse rather than +> reinvent. It also renames domain concepts in the UI: **Subscription → Integration**, +> **Document → Information type**, **Xchange → Exchange**. Backend names are unchanged. + +## 1. Where we are today + +Bus-triggered ingestion today has exactly one transport: the *internal* SW.Bus RabbitMQ +connection that Bitween shares with the microservices it is deployed next to. + +| Piece | File | Role | +|---|---|---| +| `Document.BusEnabled` / `BusMessageTypeName` | `SW.Bitween.Api/Domain/Document/Document.cs` | marks a document as bus-ingestible and names the message type | +| `BusService : IConsume` | `SW.Bitween.Api/Services/BusService.cs` | dynamic multi-message consumer; maps message type → documentId, then `XchangeService.SubmitFilterXchange` | +| `BusGateway` / `BusGatewayRoute` | `SW.Bitween.Api/Domain/Gateway/` | routing table over a bus-enabled document: (filter, subscription, partner) triples | +| `FilterService.Filter` | `SW.Bitween.Api/Services/FilterService.cs` | evaluates promoted properties → `GatewayHits` | +| runtime topology refresh | `IBroadcast.RefreshConsumers()` → `ConsumersService.RefreshConsumers()` | re-runs consumer discovery and attaches/updates queues without restart | + +Two observations that shape the design: + +1. **`BusGateway` has no connection concept at all** — it is pure routing. Adding a + `DataSource` reference is therefore additive, and `null` can keep meaning "internal bus". +2. **The runtime-mutability machinery already exists and works**: `IBroadcast` + + `IListen` over the per-node exchange, plus `IInfolinkCache` + `RevokeCacheMessage`. + We should reuse it rather than invent a second control plane. + +Relevant SW.Bus facts (verified in `SW-Bus/`): + +- `IBroadcast.Broadcast` publishes to `busOptions.NodeExchange` (direct) with a + **single shared routing key** `NodeRoutingKey`; each node declares its own + `NodeQueueName = "{NodeExchange}:{NodeId}"` as `exclusive: true, autoDelete: true` + and binds it to that key. So a broadcast fans out to *every* live node, and there is + currently **no per-node addressing**. Per-node targeting must be done by filtering + inside the listener (`if (msg.TargetNodeId != null && msg.TargetNodeId != myNodeId) return;`). +- `IListen` handlers are discovered once at startup (`ConsumerDiscovery.LoadListeners`), + run with prefetch 1 on the node channel, and have retry/dead-letter (`ListenRetryCount`). +- The exclusive-queue behaviour the leader election idea relies on is already proven in + this codebase — the node queue *is* an exclusive queue tied to one connection. + +## 2. Verdict on the idea + +The shape you proposed is right. Four things I'd change or make explicit: + +1. **Do not build one universal broker model.** A "super-entity" that has fields for + exchanges *and* topics *and* consumer groups *and* partitions *and* visibility timeouts + collapses under its own weight by the third provider. Instead: a **narrow core contract** + (subscribe / ack / publish over a normalized envelope), a **declared capability set**, + and a **provider-owned config schema** stored as JSON. Bitween core never learns what an + exchange is; providers do. +2. **Leader election is a property of the bus, so put the abstraction in the bus.** + SW.Bus is RabbitMQ-only and heavily used internally; there is one real implementation + (exclusive queue) and no appetite for a second right now. But the *seam* is worth having + from day one so a future non-Rabbit bus can supply its own primitive without touching + Bitween. Concretely: contract in `SimplyWorks.Bus.RabbitMqExtensions`, implementation in + `SimplyWorks.Bus`, and Bitween depends only on the contract (§6.1). Exclusive-queue + election gives *liveness*, not consensus — under a partition two nodes can briefly both + believe they lead — so the **fencing token stays in Bitween's database**, and leader-only + writes are guarded by it. Same trick `RunFlagUpdater.MarkAsRunning` already uses. +3. **Per-node on/off should be declarative placement, not imperative toggles.** Model + *desired* placement on the gateway (`All` / `Leader` / `Tagged` / `Explicit`), and store + manual per-node toggles as **overrides** on top of it. Otherwise a node restart silently + loses the operator's intent, or worse, silently regains a subscription they turned off. +4. **Ack ownership: persist then ack, immediately.** Never hold a broker message open while + the mapper/handler pipeline runs. Ingest = write the `Xchange` + commit, then ack. All + redelivery is then owned by the existing `DelayedRetry` / `RetryPolicy` subsystem, which + is uniform across providers — instead of nine different broker redelivery semantics. + +## 3. Domain model + +Three new entities (+ two columns on existing ones). + +### 3.1 `DataSource` — a connection to a broker + +```csharp +public class DataSource : BaseEntity, IAudited +{ + public string Name { get; set; } + public string ProviderKey { get; set; } // "rabbitmq" | "kafka" | "sqs" | "eventhub" | "mqtt" | "pulsar" + public JsonDocument Settings { get; set; } // provider-defined; secret fields via SettingsProtector (§11.1) + public bool Inactive { get; set; } + public string ConfigHash { get; set; } // set on save; drives reconciliation diffing + // health / observability + public DateTime? LastConnectedOn { get; set; } + public string LastException { get; set; } + public int ConsecutiveFailures { get; set; } +} +``` + +`Settings` is validated against the provider's descriptor (§4.2) at write time, so a bad +Kafka `bootstrap.servers` is rejected by the API, not discovered at 3am by the supervisor. + +> Naming: `DataSource` is fine and matches how you framed it, but note it is bidirectional +> (ingress *and* egress). `DataSource` + a child `DataSourceEndpoint` reads better than +> overloading one row with both directions. + +### 3.2 `DataSourceEndpoint` — one subscribe-able / publish-able thing on a data source + +```csharp +public class DataSourceEndpoint : BaseEntity, IAudited +{ + public int DataSourceId { get; set; } + public string Name { get; set; } + public EndpointDirection Direction { get; set; } // Inbound | Outbound | Both + public JsonDocument Binding { get; set; } // provider-defined: queue+exchange+routingKey, + // or topic+consumerGroup, or queueUrl, or + // hub+consumerGroup+checkpointStore, or topicFilter+qos + public JsonDocument Topology { get; set; } // optional declarative "make this exist" plan + public bool AutoProvision { get; set; } // run EnsureTopology on start +} +``` + +Splitting endpoint from data source is what makes "connect to a client's RabbitMQ and create +five queues on it" a first-class operation rather than five copies of a connection string. + +### 3.3 `ClusterNode` — who is alive, who leads, what each node runs + +```csharp +public class ClusterNode // PK = NodeId (BusOptions.NodeId, stable per process) +{ + public string NodeId { get; set; } + public string MachineName { get; set; } + public string Version { get; set; } + public string[] Tags { get; set; } // from config: Bitween:NodeTags + public DateTime StartedOn { get; set; } + public DateTime LastHeartbeatOn { get; set; } + public bool GatewaysEnabled { get; set; } // node-wide kill switch + public JsonDocument Overrides { get; set; } // { "gateway:12": false, "dataSource:3": false } +} + +public class ClusterLeader // single row, id = 1 +{ + public string NodeId { get; set; } + public long Term { get; set; } // fencing token, ++ on each acquisition + public DateTime AcquiredOn { get; set; } + public DateTime RenewedOn { get; set; } +} +``` + +### 3.4 Changes to existing entities + +- `BusGateway`: `+ int? DataSourceId`, `+ int? EndpointId`, `+ GatewayPlacement Placement` + (`All | Leader | Tagged | Explicit`), `+ string[] PlacementTags / string[] PlacementNodeIds`, + `+ bool Inactive`. **`DataSourceId == null` keeps today's exact behaviour** (internal bus + via `BusService`) — zero migration risk for existing installs, including Traxis-style ones. +- `Subscription`: optional `+ int? ResponseEndpointId` — generalizes today's + `ResponseMessageTypeName` bus publish so a subscription's output can land on an external + Kafka topic / SQS queue (§6.3). + +### 3.5 Ingress equivalence, and what it buys + +An external broker, an API gateway call, and a scheduled receiver are **the same kind of thing**: +a producer that persists an `Xchange` and lets the internal bus hand it off. Nothing about the +mapper, handler, or filter is coupled to how the message arrived. Two consequences worth stating +because they shrink the design: + +1. **The external broker's load balancing is irrelevant to Bitween's processing scale-out.** Once + ingest commits, work distribution across nodes is done by the *internal* RabbitMQ work-group + queues. Kafka consumer groups, Rabbit competing consumers, and SQS long polling only decide + **who reads from the external broker** — not who processes. So §6's placement and leader + election govern **connection ownership only**. That is a much smaller claim than it first + appears, and it means a single-node-owned Kafka subscription still processes across the whole + cluster. +2. **The internal RabbitMQ stays a hard dependency**, even for a Kafka-only or SQS-only client. + Worth being explicit with ops: adding external brokers does not let anyone drop SW.Bus. + +### 3.6 Filtering: it already exists, and there is a second kind worth adding + +**Bitween-side filtering is unchanged and already provided.** `FilterService` extracts the +Document's promoted properties (JSON/XML paths) and evaluates `BusGatewayRoute.MatchExpression` +per route; a null expression matches everything. External ingress inherits this for free — that is +the whole reason to route through `BusGateway` rather than invent a parallel concept. Nothing to +build. + +But note *where* it happens: **after** the payload has been uploaded to cloud storage and the +`Xchange` committed. For a client topic where only 5% of messages are interesting, that is 95% of +the blob PUTs, rows, and queue traffic spent to discover the message was irrelevant. So there is a +second, cheaper filter worth having: + +| | Broker-side selection | Bitween-side filtering | +|---|---|---| +| Lives in | `DataSourceEndpoint.Binding` | `Document.PromotedProperties` + `BusGatewayRoute.MatchExpression` | +| Runs | before Bitween sees the message | after persist, at hop 1 | +| Cost of a non-match | zero | one blob PUT + one row + one queue hop | +| Expressiveness | whatever the broker offers | full match expressions over promoted properties | +| RabbitMQ | binding routing keys, headers exchange with `x-match` | — | +| MQTT | topic filters with `+`/`#` | — | +| Kafka | **nothing** — no server-side filtering exists | — | + +So broker-side selection is a **capability** (`SupportsServerSideSelection`), not a guarantee. Where +it exists, prefer it and let Bitween-side filtering handle what the broker can't express. Where it +doesn't (Kafka), the volume lands on Bitween and `Document.DisregardsUnfilteredMessages` becomes the +tool that stops unmatched messages from accumulating. + +### 3.7 What lives in `DataSource` vs `DataSourceEndpoint` vs `BusGateway` + +The boundary is **transport vs business meaning**, and it should stay that clean: + +| Concern | Lives in | Examples | +|---|---|---| +| How to reach the system | **`DataSource`** | hosts, vhost/cluster, credentials, TLS, auth mechanism, client id, management URL, pass-through config | +| Health and connection state | **`DataSource`** | `LastConnectedOn`, `LastException`, `ConsecutiveFailures`, `ConfigHash` | +| Which object on that system, and how to read/write it | **`DataSourceEndpoint`** | queue/topic/table/collection name, prefetch, consumer group, declare mode, topology plan, broker-side selection | +| Where the cursor is | **`DataSourceEndpointCursor`** (§3.8) | Kafka offset, Mongo resume token, RDBMS high-water mark | +| **What the message means** | **`BusGateway`** | `DocumentId` — the information type | +| **Who processes it, with which partner, under which filter** | **`BusGatewayRoute`** | `SubscriptionId`, `PartnerId`, `MatchExpression` | +| Placement and enablement | **`BusGateway`** | `Placement`, `WorkGroupId` (§5.1a), `Inactive` | + +The join is `BusGateway.EndpointId` + the existing `BusGateway.DocumentId`. `DataSource` and +`DataSourceEndpoint` carry **no business semantics whatsoever** — that is what makes them reusable +for egress, for enrichment, and for the non-messaging kinds below. + +**One gap this exposes: mixed-type topics.** A queue usually carries one message type, so +"endpoint → one document" is fine. A Kafka topic often carries several. So `BusGateway` needs a +**document resolution strategy** rather than only a fixed `DocumentId`: + +``` +DocumentResolution = Fixed(documentId) + | Header(headerName → document code) // Rabbit headers, Kafka headers + | RoutingKey(pattern → document code) // Rabbit + | PayloadPath(jsonPath/xpath → document code) +``` + +`Fixed` covers phase 2; the rest are additive and cheap *if the column exists from the start*. + +### 3.8 Keeping `DataSource` reusable beyond messaging + +This matters now, because the abstractions ship as a NuGet package and renaming a published +contract later is a breaking change. Three decisions to take up front — all cost nothing today: + +**1. Name and shape it around *data sources*, not messaging.** `IDataSourceProvider` is the wrong +name. Use `IDataSourceProvider` with a declared kind: + +```csharp +public enum DataSourceKind { Broker, Relational, Document, ObjectStore, FileTransfer, Http } +``` + +**2. Split behaviour into capability interfaces instead of one fat contract.** A provider +implements only what its system can do; core checks for the interface rather than assuming: + +| Capability | Meaning | Broker | RDBMS | NoSQL | +|---|---|---|---|---| +| `ISubscribeCapable` | push/streaming delivery | ✔ | — | Mongo change streams ✔ | +| `IPollCapable` | pull on a schedule, cursor-driven | — | ✔ | ✔ | +| `IPublishCapable` | write a message/row/document out (egress, §6.3) | ✔ | ✔ (insert) | ✔ | +| `IQueryCapable` | ad-hoc read for **enrichment during mapping** | — | ✔ | ✔ | +| `ITopologyCapable` | create/inspect objects | ✔ | ✔ (DDL, usually off) | ✔ | +| `IBrowseCapable` | discovery (§1.12) — topics, tables, collections | ✔ | ✔ | ✔ | + +`IQueryCapable` is the one that is easy to miss and changes the design: a `DataSource` may be +consumed by a **mapper** rather than acting as an ingress at all — "look up this SKU in the +client's SQL Server while mapping". That means data sources must be resolvable from adapter +context alongside `__partner__` and `__globals__` (say `__datasources__`), which is a contract +decision, not an implementation detail. + +**3. Add the cursor concept now.** Push sources track position in the broker; pull sources cannot. +RDBMS high-water marks, Mongo resume tokens, SFTP "seen files", and Kafka offsets are all the same +idea, and Bitween already needs somewhere durable and per-node-agnostic to keep it: + +```csharp +public class DataSourceEndpointCursor // one row per endpoint +{ + public int EndpointId { get; set; } + public string Kind { get; set; } // "offset" | "timestamp" | "token" | "id" + public string Value { get; set; } + public DateTime UpdatedOn { get; set; } + public long Term { get; set; } // fencing (§6.1) — only the current owner may advance it +} +``` + +Retrofitting this after three providers exist is painful; adding the table now is a migration. + +**Where this pays off immediately:** Bitween *already* has pull-based ingestion — `ReceivingJob`, +Quartz schedules, and native receivers for S3, FTP, POP3, and Azure Blob. Today each of those +carries its own connection details inside `Subscription.ReceiverProperties`, so an FTP credential +is duplicated per subscription, unencrypted beyond the adapter's own handling, untestable, and +unbrowsable. If `DataSource` is designed transport-agnostically, those receivers can later +**reference a `DataSource`** instead — one encrypted, health-monitored, test-connectable, +discoverable connection reused across subscriptions. That unification, not the brokers, is the +strongest long-term argument for getting this entity right on the first attempt. + +Scope note: only the **abstraction** is in scope now. No RDBMS or NoSQL provider is being built; +the point is that adding one later should require no change to `DataSource`, `DataSourceEndpoint`, +the registry, placement, health, or the UI shell. + +## 4. Plugin architecture + +### 4.1 Packaging + +New abstraction-only NuGet package **`SW.Bitween.DataSources.Abstractions`** (mirrors how +`SW.Bus.RabbitMqExtensions` has no `RabbitMQ.Client` dependency). Provider packages depend +only on it: + +``` +SW.Bitween.DataSources.Abstractions ← contracts, descriptors, envelope, capabilities +SW.Bitween.DataSources.RabbitMq ← built-in, in-tree +SW.Bitween.DataSources.Kafka ← built-in, in-tree +SW.Bitween.Messaging.Sqs / .EventHubs / .Mqtt / .Pulsar ← separate, opt-in +``` + +Loading is **startup-only** (as you specified): `services.AddBitweenDataSources()` registers +in-tree providers, then for each directory in `Bitween:MessagingProviderPaths` creates an +`AssemblyLoadContext` per plugin folder with `SW.Bitween.DataSources.Abstractions` (and +`Microsoft.Extensions.*`) resolved from the **host**, everything else from the plugin folder. +This gives dependency isolation — critical, since `Confluent.Kafka`, `AWSSDK`, and +`Azure.Messaging.EventHubs` all drag in conflicting transitive versions. Discovered +`IDataSourceProvider` implementations land in a singleton `IDataSourceProviderRegistry` +keyed by `ProviderKey`. + +> **Rejected alternative:** running providers as SW.Serverless adapters (subprocess + +> stdin/stdout, distributed via cloud storage). It is a great fit for mappers/handlers, and a +> bad fit here: providers hold long-lived connections, need sub-millisecond ack round-trips, +> and must surface streaming callbacks — none of which survive a request/response RPC over +> stdio. In-process ALC plugins, loaded at startup, is the right trade. + +### 4.2 The contract + +The whole point is that **each broker's concepts stay inside its provider**. Core sees: + +```csharp +public interface IDataSourceProvider +{ + string Key { get; } + ProviderDescriptor Describe(); // capabilities + config schema (drives UI *and* validation) + Task Connect(ResolvedConfig settings, CancellationToken ct); +} + +public interface IDataSourceConnection : IAsyncDisposable +{ + Task CheckHealth(CancellationToken ct); + Task Subscribe(ResolvedConfig binding, SubscribeOptions options, + Func> onMessage, + CancellationToken ct); + Task Publish(ResolvedConfig binding, OutboundMessage message, CancellationToken ct); + ITopologyManager Topology { get; } // null when !Capabilities.CanManageTopology +} + +public interface IMessageSubscription : IAsyncDisposable +{ + SubscriptionState State { get; } // Starting | Running | Degraded | Stopped + event EventHandler Faulted; +} + +public interface ITopologyManager +{ + Task Plan(TopologyPlan plan, CancellationToken ct); // dry run — show the operator + Task Apply(TopologyPlan plan, CancellationToken ct); // idempotent create/bind + Task> Browse(BrowseQuery query, CancellationToken ct); // pickers in the UI +} +``` + +Normalized envelope — the lowest common denominator that all six brokers actually have: + +```csharp +public sealed record InboundMessage( + ReadOnlyMemory Body, + IReadOnlyDictionary Headers, // emulated via message attributes / properties where needed + string Key, // Kafka key / MQTT topic / Rabbit routing key / SQS group id + string ProviderMessageId, + DateTimeOffset? Timestamp, + IReadOnlyDictionary ProviderMetadata); // offset, partition, receiptHandle, deliveryTag… + +public enum AckDecision { Ack, Reject, RequeueLater, DeadLetter } +``` + +`ProviderMetadata` is the escape hatch: providers can surface anything, and it is carried +into the `Xchange` references / adapter context so a mapper can read `partition` or +`receiptHandle` without core knowing they exist. + +### 4.3 Capabilities: how the complexity is actually contained + +```csharp +public sealed record ProviderCapabilities( + bool CanManageTopology, bool CanBrowseTopology, + bool SupportsHeaders, bool SupportsOrderingKey, bool SupportsPartitions, + bool SupportsNack, bool SupportsRequeueDelay, bool SupportsNativeDeadLetter, + bool SupportsExclusiveConsumer, // → can back leader election / single-active-consumer + bool SupportsCompetingConsumers, // → Placement=All is safe + bool SupportsTransactions, bool SupportsBatch, + int MaxMessageBytes); +``` + +Rules that follow mechanically from this, enforced in one place: + +- `Placement = All` is only offered when `SupportsCompetingConsumers` (Kafka consumer + groups: yes; a single MQTT non-shared subscription: no → forced to `Leader`). +- `AckDecision.RequeueLater` falls back to "ack + `DelayedRetry` row" when + `!SupportsRequeueDelay` — so behaviour is uniform even where the broker can't do it. +- `SupportsNativeDeadLetter == false` → dead letters are recorded as failed `Xchange`s only. +- The UI never renders a control the provider didn't declare — no per-provider UI code. + +`ProviderDescriptor` also carries the **config schema** (field key, label, type, required, +`IsSecret`, default, enum options, group, help text, plus optional dependsOn) for both +connection settings and bindings. Both API validation and the UI form are generated from it, +which is what makes adding a provider a zero-UI-change operation. + +## 5. Runtime: supervisor and reconciliation + +``` +MessagingSupervisor (IHostedService, singleton) + desired = f(DB: active DataSources × active BusGateways × placement × node overrides × leadership) + actual = in-memory map { dataSourceId → IDataSourceConnection, gatewayId → IMessageSubscription } + Reconcile() → diff → open/close connections, start/stop subscriptions (idempotent, lock-guarded) +``` + +Reconcile is triggered by: + +| Trigger | Mechanism | +|---|---| +| startup | after `SchedulerSeedService`-style hosted-service start | +| config change (CRUD on DataSource / Endpoint / BusGateway / Subscription) | `IBroadcast.Broadcast(new GatewayControlMessage { Action = Reconcile })` from the CQAPI handler — same pattern as `Documents/Update.cs` calling `RefreshConsumers()` | +| operator start/stop on one node | same message with `TargetNodeId` set; listener no-ops if it isn't me | +| leadership change | `ILeaderElector` raises `LeadershipChanged` → local reconcile | +| drift / missed message | periodic Quartz job (`ClusterHeartbeatJob`, default every 15–30s) also calls `Reconcile()` | +| connection fault | `IMessageSubscription.Faulted` → backoff (exponential, capped) then reconcile that entry only | + +Diffing granularity uses `ConfigHash`: connection-settings change → rebuild connection and +its subscriptions; binding-only change → restart just that subscription; placement change → +start/stop only. + +Ingest path per message (deliberately thin): + +``` +InboundMessage + → IngressPipeline: decode (provider-declared content type), size guard, optional dedupe + → XchangeService.SubmitFilterXchange(document.Id, xchangeFile, references, correlationId) + → SaveChanges → return AckDecision.Ack +``` + +Everything downstream — `FilterService` gateway routes, partner values, mappers/handlers, +`RetryPolicy` — is **completely unchanged**. That is the main reason to route external +brokers through `BusGateway` rather than inventing a parallel concept. + +### 5.1 The ingest contract, stated exactly + +The provider's entire job is: **persist the message, let the existing event mechanism hand it to a +work group, ack.** No filtering, no mapping, no handler — none of the pipeline runs while the +broker message is open. Verified end-to-end on v2: + +| # | Step | Code | +|---|---|---| +| 1 | provider → `SubmitFilterXchange(documentId, file, refs, correlationId)` | `XchangeService.cs:68` | +| 2 | payload uploaded to **cloud storage**, `Xchange` added to the change tracker | `CreateXchange` → `AddFile` → `_cloudFiles.WriteTextAsync` (`:340`) | +| 3 | `SaveChangesAsync()` — the commit | `XchangeService.cs:85` | +| 4 | **after** commit, the domain event is published to the work group's queue | `BitweenDbContext.SaveChangesAsync:399-407` → `publish.Publish(hasWorkGroup.GetBusMessageName(), {Id})` | +| 5 | `XchangeService` (`IConsumeExtended`, one queue per work group, prefetch/priority from `WorkGroup.Options.RabbitMqOptions`) consumes → filter → mapper → handler → result → retry | `Process(XchangeMessage)` | +| 6 | provider acks the broker message | — | + +Three qualifications on "the right work group handles it": + +**(a) It is two hops, and the first one lands on a queue you currently cannot tune.** The +filter-path `Xchange` is created with `workGroup: null`, and `Xchange.cs:42` resolves that to +`WorkGroup.None` — a static, **transient** instance (`new() { BusMessageName = "Ungrouped" }`, +`Id = 0`, no DB row, `Options == null`). So hop 1 routes to `0Ungrouped`, whose `ConsumerOptions` +resolve to `null` prefetch and priority and therefore fall back to +`BitweenOptions.BusDefaultQueuePrefetch`. Hop 1 runs `FilterService` and creates the +per-subscription Xchanges (`CreateXchangesForHits`); only *those* carry +`subscription.WorkGroup` and land on a tunable queue. + +Net effect: **all external ingest funnels through one shared, non-configurable queue** before it +ever reaches a work group — because there is no `WorkGroup` row for `None` to edit in the UI. + +Three ways out, in increasing order of value: + +1. Monitor `0Ungrouped` deliberately (the WorkGroups search already surfaces live queue stats from + the management API, so the data path exists). +2. Set the document's `DisregardsUnfilteredMessages`, which makes `SubmitFilterXchange` filter + **inline** and skip hop 1 entirely — slower ack, no shared queue. +3. **Recommended: add `BusGateway.WorkGroupId`** and pass it through to the filter-path + `CreateXchange(document, workGroup, …)` overload, which *already accepts a work group* and is + only ever called with `null`. That makes external ingest a first-class, tunable queue with its + own prefetch and priority, isolates one client's broker traffic from another's, and is a + genuinely small change: one column, one argument, no new concepts. + +**(b) The event is published *after* commit, so there is a dual-write gap — and acking makes it +consequential.** If the publish fails (broker blip, process killed between commit and publish), +`SaveChangesAsync` throws: the row is committed, nothing will ever process it, and the provider +does **not** ack — so the broker redelivers and creates a *second* `Xchange` for the same message. +Net result: one orphan plus one processed copy. Today the same gap exists for API-created +Xchanges, but an HTTP caller gets a 202 and notices; a broker has nobody to notice. **There is no +orphan sweeper in the codebase** (verified). Two fixes, cheapest first: + +- **Sweeper (recommended, ~50 lines):** a Quartz job that finds Xchanges with no `XchangeResult` + older than N minutes and republishes the trigger event. Reuses `RetryJob`'s shape and fixes + *every* ingress path, not just brokers. +- **Outbox for the trigger event:** the same machinery as the egress outbox (§6.3) — correct, but + 6b-sized. + +This is also the concrete reason the dedupe question (§12.3) matters: with a provider message id +as the dedupe key, redelivery after a failed publish is idempotent instead of duplicating. + +**(c) Ack latency includes a blob upload, not just a DB insert** — step 2 is a network round trip to +S3/Azure/Oracle. But see §5.2: measured in production, the whole ingest-plus-plumbing path is +~330 ms p50, and it is **not** the bottleneck. The bottleneck is adapter execution, by two orders +of magnitude. + +### 5.2 Measured baseline — Traxis production (Bitween **6.1**), 2026-07-30 + +Read-only measurements against `traxis_prod` (`infolink` schema, DigitalOcean managed PG 17, +CloudFiles = DO Spaces `nyc3`). + +> **This is a 6.1 install, not 8.x — but the pipeline logic is substantially the same.** +> `Xchange` created → payload to cloud storage → domain event published after commit → consumed +> → filter → mapper → handler → `XchangeResult`: that shape, and therefore its per-message cost +> structure, is unchanged between 6.1 and 8.x. What 8.x added is **routing granularity** (work +> groups replacing per-event-type queues), **native adapters**, **auto-retry**, and **gateways** — +> none of which alter the cost of the steps that were measured. So the numbers are broadly +> transferable; the exceptions are specific and worth naming. +> +> Latest applied migration is `20230910151704_SubscriptionCategory` (Sept 2023), and +> `releases/r6.1` is still the `SW.Infolink.*` generation. Verified absent, in both the live schema +> and the r6.1 tree: +> **work groups** (no `work_group` table, no `work_group_id` column, zero WorkGroup source files), +> **native adapters** (zero source files), and the **auto-retry subsystem** (no `delayed_retry`, +> `retry_policy`, or `group_attempt_counts`; only `xchange.retry_for`). There are no +> `bus_gateway`/`api_gateway` tables either. +> +> | Finding | Transfers to 8.x? | +> |---|---| +> | Volume, arrival rate, payload sizes | **Yes** — properties of the client's business traffic, not of Bitween | +> | Handler vs no-handler latency ratio (~100×) | **Yes, qualitatively** — it is an A/B inside one system, so the comparison is sound | +> | The 0.33 s "plumbing" figure | **Yes as a cost estimate** — same blob PUT, same insert/commit, same publish-after-commit, same consume. What differs is only *which* queue absorbs it: 6.1 used the **legacy per-event-type queues** (`InternalXchangeCreatedEvent` etc., the path `ConsumeLegacyEventMessages` still exists to preserve), 8.x uses work-group queues. Routing granularity changed; per-message cost did not | +> | "Zero native adapters" | **Reframed, not invalidated** — native adapters don't exist in 6.1, so this is unavailability rather than a choice. It makes the 32 s structural *there*, and makes the biggest lever something the 8.x upgrade newly unlocks | +> | Two-hop / `0Ungrouped` analysis (§5.1a) | **Not evidenced here at all** — it rests solely on reading the v2 source, which I verified directly | +> | Index sizes | Indicative only — 6.1-era index set on a 3-year-old schema | +> +> Bottom line: treat the volume, payload, and latency-ratio findings as real inputs for 8.x +> planning. Re-measure the absolute service time once an 8.x install with native adapters exists, +> because that is the one constant the upgrade is expected to move — by a lot. + +**Volume (a genuinely small workload):** + +| Metric | Value | +|---|---| +| Data span | 2025-10-01 → 2026-07-30 (10 months) | +| Total xchanges | 31,078 (≈100/day average) | +| Recent daily range | 45 – 4,128/day; median ≈ 1,800 | +| **Peak minute** | **140 xchanges/min ≈ 2.3/s** | +| Input payload | avg 3.6–17.5 KB (typically ~5 KB), **max 1.08 MB** | +| Storage | `xchange` 560 MB heap / **1,632 MB indexes**; `xchange_promoted_properties` 1,206 MB / 1,215 MB; `xchange_result` 413 MB / 360 MB — for 31k rows, with `n_tup_del = 0` | +| Adapter mix (7 d) | 11,538 of 14,468 xchanges have a handler; **`native.*` adapters in use: 0** | + +**Latency, and this is the finding that matters** (7-day window, `started_on` → `result.finished_on`): + +| Path | n | p50 | p90 | +|---|---|---|---| +| **without** a handler | 2,930 | **0.33 s** | 0.71 s | +| **with** a handler | 11,538 | **32.67 s** | 78.29 s | +| combined | 14,468 | 22.80 s | 70.35 s (p99 179 s, max 253 s) | + +The no-handler row *is* the full Bitween plumbing — blob PUT, insert, commit, publish, two queue +hops, filter, result — and it costs **330 ms p50**. The handler adds **~32 seconds**. With zero +native adapters in use, every handler is a **serverless adapter, i.e. a spawned .NET subprocess per +invocation**: process start, assembly extraction and load, then the actual external call. + +**So the ceiling is adapter execution concurrency — not storage, not the database, and nowhere near +the broker.** (This corrects an earlier claim in this document that storage and the DB would be the +limit; measured, they are ~1.5% of the time budget.) + +**Capacity model.** Concurrency required is Little's Law, `L = λW`: + +| Scenario | Arrival λ | Service W | **Concurrent in-flight** | +|---|---|---|---| +| Traxis peak today | 2.3/s | 32.7 s | **≈ 76** | +| A client at 10× | 23.3/s | 32.7 s | **≈ 762** | +| 10× **if** service time drops to 1 s | 23.3/s | 1 s | **≈ 24** | + +`BitweenOptions.BusDefaultQueuePrefetch` is **12**, so concurrency per node per work-group queue is +about 12. 762 in-flight would need ~64 node-queue slots — and each in-flight handler is a +*subprocess*, so "just raise prefetch to 60" means 60 concurrent subprocesses per node, which is +not viable. Three conclusions: + +1. **Cutting adapter service time is the highest-leverage change for 10× scale, by far** — native + adapters (currently unused here) or a warm/pooled serverless process. Going from 32 s to 1 s + turns 762 required slots into 24. +2. **Broker prefetch is not the lever.** Ingest acks in ~330 ms; at 5 KB payloads a prefetch of 20 + is 100 KB of memory. Bound `prefetchCount` by the **max** payload (1 MB here ⇒ prefetch 500 + would be 500 MB), not the average, and otherwise don't be timid. +3. **The work-group queues are the shock absorber, and that is correct** — acking fast and letting + the queue grow is exactly what stops a 33-second handler from blocking a client's broker. Which + makes queue depth (`QueueBackpressureThreshold`, default 5000) the alarm that matters, and makes + `BusGateway.WorkGroupId` (§5.1a) more valuable, not less: external ingest needs its own tunable + queue rather than sharing the untunable `0Ungrouped`. + +**Also worth acting on independently:** indexes on these three tables total **~3.2 GB against +2.1 GB of heap for 31k rows**, with no deletes recorded — a `REINDEX CONCURRENTLY` should reclaim +most of it, and it will matter more at 10×. + +### 6.1 Leader election + +Election lives in **SW.Bus**, not in Bitween. The verified layering forces this and happens +to be the right design anyway: + +- `SW.Bitween.Api` references only `SimplyWorks.Bus.RabbitMqExtensions` (8.1.11) — the + contracts package with **no `RabbitMQ.Client` dependency**. `SimplyWorks.Bus` (which has + it) is referenced only by `SW.Bitween.Web`. Implementing the elector inside + `SW.Bitween.Api` — where the supervisor and Quartz jobs live — would mean giving it a + broker dependency it deliberately doesn't have. +- SW.Bus already owns everything the elector needs: the `ConnectionFactory`, a stable + `BusOptions.NodeId`, the `{env}.{app}` naming scheme, and connection-shutdown events. +- Whoever supplies the bus supplies the primitive. Swap the bus later and the new bus brings + its own election; Bitween's code is unchanged. That is the "dynamic" part, obtained for + free by placing one interface in the right assembly — not by writing a second + implementation now. + +**Contract** → `SimplyWorks.Bus.RabbitMqExtensions`: + +```csharp +public interface ILeaderElection +{ + string Scope { get; } // election is per named role, see below + string NodeId { get; } + bool IsLeader { get; } + int Epoch { get; } // local acquisition counter — liveness only, NOT a fence + event EventHandler LeadershipChanged; +} + +public interface ILeaderElectionFactory +{ + ILeaderElection GetOrCreate(string scope); +} + +public sealed record LeadershipChangedEventArgs(string Scope, bool IsLeader, int Epoch, DateTimeOffset OccurredOn); +``` + +**Scoped election matters, and should be in the contract from day one.** A single global +leader means one node performs *all* external ingest while the others idle — for a client +with twelve Kafka topics that is a deliberate bottleneck. With per-scope election +(`"gateway:{id}"` or `"datasource:{id}"`) each gateway is independently owned, so leadership +spreads across nodes naturally and a node loss only redistributes its share. Cost in +RabbitMQ: one extra exclusive queue per scope on one shared connection — negligible. + +**Implementation** → `SimplyWorks.Bus`, `RabbitExclusiveQueueElection`, registered by an +opt-in `services.AddBusLeaderElection()`: + +- One dedicated `IConnection` for all election scopes, separate from the consumer connection + so consumer reconnect churn can never drop leadership. +- Per scope, loop `QueueDeclare("{ProcessExchange}.{ApplicationName}.leader.{scope}", + durable: false, exclusive: true, autoDelete: true)`. Success ⇒ leader for as long as that + connection lives. `RESOURCE_LOCKED` (405) ⇒ follower; retry every ~5s with jitter. +- On connection shutdown: raise `LeadershipChanged(false)` **before** attempting + reacquisition, so leader-only work stops first. + +**Fencing stays in Bitween's database.** RabbitMQ cannot hand out a monotonic counter, and +`Epoch` is process-local, so it is not a valid fence. On each `LeadershipChanged(true)` +Bitween does one statement per scope: + +```sql +UPDATE cluster_leader SET node_id = @me, term = term + 1, acquired_on = now(), renewed_on = now() +WHERE scope = @scope RETURNING term; +``` + +That returned `term` is the fence: every leader-only write is guarded by +`WHERE term = @myTerm`. Clean split — **the bus provides the lock, the database provides the +fence** — and it means the fencing guarantee is identical no matter which bus supplies +election later. Combined with the existing `RunFlagUpdater` row-level mutex, split-brain +degrades to a liveness blip rather than a correctness bug. + +**Default when election isn't registered:** `SingleNodeLeaderElection` in Bitween — always +leader, still takes a `term` from the DB. This is not a second real implementation; it exists +so single-node installs, `dotnet run`, and the integration fixture behave deterministically +without election timing in the test path. + +**Sequencing note:** this needs a PR to the public `SW-Bus` repo (CI publishes +`SimplyWorks.Bus*` on merge to `main`) and a version bump from 8.1.11 in both +`SW.Bitween.Api.csproj` and `SW.Bitween.Web.csproj`. Phases 0–2 don't depend on election, so +that PR can land in parallel and only gates phase 3. + +### 6.2 Placement and per-node enable/disable + +Effective decision for (gateway, node): + +``` +run = dataSource.Active + && !gateway.Inactive + && node.GatewaysEnabled + && node.Overrides["gateway:{id}"] != false + && placement matches (All | Leader && IsLeader | Tagged && node.Tags ∩ tags | Explicit && id ∈ nodes) +``` + +Admin API: `POST /cluster/nodes/{nodeId}/gateways/{gatewayId}/{enable|disable}`, +`POST /cluster/nodes/{nodeId}/{drain|resume}`, `GET /cluster` (nodes, leader, per-node +running subscriptions and their state). Each mutation writes DB **then** broadcasts — DB is +the source of truth, the broadcast is only a latency optimization, so a node that was down +during the broadcast picks the change up from its next heartbeat reconcile. + +### 6.3 Egress — review + +#### What exists today (verified in code) + +Outbound is **not** absent, but it is unmodeled and it has a durability gap: + +| Location | Behaviour | +|---|---| +| `XchangeService.cs:418-421` (v2; `:413` on r8.0) | `if (!string.IsNullOrWhiteSpace(xchange.ResponseMessageTypeName) && responseFile != null && !responseFile.BadData) await _publish.Publish(xchange.ResponseMessageTypeName, responseFile.Data);` — raw string publish to the internal bus, **inline, before the `SaveChangesAsync()` that commits the `XchangeResult`** | +| `BitweenDbContext.SaveChangesAsync:399-407` | domain events published **after** `base.SaveChangesAsync` — commit first, publish second | + +Both are **dual writes with no outbox**, in opposite directions: + +- `ResponseMessageTypeName` publishes *before* the `XchangeResult` commits. Crash in between ⇒ + the downstream system received the message, Bitween has no record of success, and a + reprocess/retry **publishes it again**. Silent duplicate. +- Domain events publish *after* commit. Broker unavailable ⇒ the `Xchange` exists and nothing + ever processes it. Silent loss. + +Neither is catastrophic today because the target is a co-located internal RabbitMQ that is +essentially always up, and duplicates on `InternalXchangeCreatedEvent` are mostly idempotent. +**Both properties disappear the moment the target is a client's broker over the public +internet.** So the review conclusion is: egress isn't a new risk introduced by this design, +it is an existing latent one whose blast radius external brokers multiply. + +#### Why it is genuinely a big change + +Ingress and egress are not symmetric, and the asymmetry is the whole difficulty: + +> **Ingress:** the broker holds the message until Bitween commits. At-least-once is free — +> just persist, then ack. +> **Egress:** Bitween holds the only copy of the outcome and must guarantee it reaches the +> broker. At-least-once has to be *built*. + +That means a **transactional outbox**, which is a new subsystem, not a handler: + +1. **Outbox table + drain job.** `OutboundMessage` (payload ref, endpoint id, envelope + metadata, dedupe key, attempt count, next-attempt-on, state) written **in the same + transaction** as the `XchangeResult`; a Quartz `OutboxJob` drains it. New table, new job, + new failure modes, plus interaction with placement/leadership (§6.2) so N nodes don't + double-drain — solvable with the same `RunFlagUpdater`-style row claim. +2. **Retry ownership collides with `RetryPolicy`.** If a publish fails, is that an `Error` on + the Xchange — which sends `DelayedRetry` back through **mapper and handler again**, re-doing + any side effect the handler already performed (an FTP upload, an HTTP POST) — or is it a + transport-local retry? It must be **transport-local**: retry the publish only. Reuse the + `DelayStrategy` shapes from `SW.Bitween.Sdk/Model/AutoRetry/` but keep outbox records + separate from `DelayedRetry`. Getting this wrong produces duplicate real-world side effects, + and it is the single subtlest point in the whole design. +3. **The envelope must be dynamic, and that is a mapping problem.** Ingress normalizes down to + `XchangeFile.Data` (a string). Egress needs topic/queue, key, headers, and sometimes + partition — usually *derived from the payload* (device id → MQTT topic, order id → Kafka + key). Static config fields can't express that. Recommendation: binding fields accept the + **`{{partner.KEY}}` / `{{globals.SET.KEY}}` token syntax adapter properties already use**, + extended with the outbound payload's promoted properties — reusing that mechanism and its + existing authoring/preview components (§11.5) rather than introducing a second expression + language. This is the piece most likely to be underestimated. +4. **Loop prevention.** Bitween consuming from broker A and publishing to broker B — where a + client has a route back — creates cycles that will be discovered in production. Stamp + `x-bitween-hop` and `x-bitween-origin-xchange` headers, enforce a configurable max hop, + reject beyond it. Trivial to add now, painful to retrofit, and impossible for brokers where + `SupportsHeaders = false` (SQS attributes work; raw MQTT 3.1.1 has no headers at all — so + for those, the loop guard has to live in the payload or be declared unavailable). +5. **Blast radius and authorization.** Egress means Bitween *writes into client systems*. + `DataSourceEndpoint.Direction` must be enforced server-side (an `Inbound` endpoint refuses + publish), every publish audited, and per-endpoint rate limits available. An operator who + configured a read-only connection to a client's production topic must not be able to + publish to it by editing one subscription field. +6. **Ordering.** A parallel outbox drain destroys per-key ordering. If any client needs it, + the drain must be single-flight per key or per endpoint — which caps throughput. This is + §12.1 again, and egress is what turns that question from theoretical to blocking. +7. **The existing feature can't silently change semantics.** Folding + `ResponseMessageTypeName` into the outbox would change delivery timing and duplicate + behaviour for every current installation, including the Traxis-style ones. Keep it working + exactly as-is; make the endpoint path **opt-in** via a new field; migrate deliberately later + with the old behaviour available as a flag. + +#### Recommended split — and it should come earlier than phase 6 + +| | Scope | Size | Guarantee | +|---|---|---|---| +| **6a. Inline publish** | `native.publishToEndpoint` handler in `SW.Bitween.NativeAdapters`, publishing to a `DataSourceEndpoint` with token-templated binding. Publish inline; failure = handler exception = normal `XchangeResult` error + existing `RetryPolicy`. No outbox. | S–M | at-most-once-ish, same as `ResponseMessageTypeName` today — **explicitly documented as such** | +| **6b. Durable outbox** | `OutboundMessage` table, `OutboxJob`, transport-local retry, dedupe key, loop guard, ordering mode, per-endpoint rate limit. `Subscription.ResponseEndpointId` routes through it. | L | at-least-once with dedupe key | + +**6a belongs right after the first provider, not at phase 6.** It is a handler — it composes +with the existing pipeline and needs no core change — and bidirectional flow is exactly what +makes the MQTT and Pulsar demos land (a demo that only *reads* from a device network is half a +demo). Ship 6a for the demos, and gate 6b on the first client who needs guaranteed delivery, +by which point you will know their ordering answer too. + +### 6.4 Topology provisioning + +- `POST /data-sources/{id}/endpoints/{id}/plan` → `TopologyDiff` (dry run shown in UI). +- `POST .../apply` → `ITopologyManager.Apply` (idempotent). +- `GET /data-sources/{id}/browse?kind=queue|topic|exchange` → picker data. +- `AutoProvision = true` runs `Apply` before `Subscribe` during reconcile — and only on the + node that is actually going to consume, guarded by leadership for `Placement = Leader`. + +## 7. Delivery plan + +Each phase is independently shippable and leaves the system working. + +| Phase | Scope | Notes | +|---|---|---| +| **0. Abstractions** | `SW.Bitween.DataSources.Abstractions` (contracts, descriptor, capabilities, envelope), `IDataSourceProviderRegistry`, ALC plugin loader, `AddBitweenDataSources()` | no behaviour change; unit tests on descriptor validation + loader isolation | +| **1. Data sources** | `DataSource` + `DataSourceEndpoint` entities, EF config, migrations for **all three** providers, CRUD resources under `SW.Bitween.Api/Resources/DataSources/`, secrets via `SettingsProtector`, RBAC permission area, `POST /test-connection`, cache + `RevokeCacheMessage` invalidation | config-only; nothing consumes yet | +| **2. Supervisor + external RabbitMQ** | `MessagingSupervisor`, reconcile loop, `GatewayControlMessage` (`IListen`), `SW.Bitween.DataSources.RabbitMq` provider, `BusGateway.DataSourceId/EndpointId`, ingest → `SubmitFilterXchange` | `DataSourceId == null` still goes through `BusService`; first end-to-end external broker | +| **3a. Election (SW-Bus PR)** | `ILeaderElection` / `ILeaderElectionFactory` in `SW.Bus.RabbitMqExtensions`, `RabbitExclusiveQueueElection` + `AddBusLeaderElection()` in `SW.Bus`, Testcontainers test that kills the leader's connection and asserts single ownership | parallel to 1–2; publishes via CI, then bump 8.1.11 in Bitween | +| **3b. Cluster** | `ClusterNode`/`ClusterLeader` (per-scope rows), `ClusterHeartbeatJob` (Quartz), `SingleNodeLeaderElection` default, term fencing, placement + overrides, `/cluster` admin API | prerequisite for anything single-consumer | +| **4. Topology** | `ITopologyManager` for RabbitMQ, plan/apply/browse endpoints, `AutoProvision` | "create queues on the client's Rabbit" | +| **5. Providers** | Kafka → Event Hubs (verify over Kafka endpoint) → MQTT → Pulsar, then demand-driven (§8.2); one PR each, none touching core | Kafka spike happens *before* phase 0 freezes the contract | +| **5b. Egress (inline)** | `native.publishToEndpoint` with token-templated bindings — **interleaved with phase 5, before MQTT** (§6.3) | at-most-once, documented as such; unlocks two-way demos | +| **6. Egress (durable)** | `OutboundMessage` outbox + `OutboxJob` + transport-local retry + loop guard + `Subscription.ResponseEndpointId` | gate on the first client needing guaranteed delivery | +| **7. Node health** | `NodeHeartbeat` process metrics on `ClusterNode`, `/ops/nodes` + `/ops/nodes/{id}/connections`, per-connection counters, budget guards (§10) | extends the existing `Ops` resource family | +| **8. UI** | `ApiClient` contract + mock first, then http; DataSources list/editor with **descriptor-driven forms** (reusing the `AdapterConfig` pattern), test-connection, endpoint editor, gateway picker, Nodes page (§11) | v2 `ClientApp` stack: TanStack Query + Tailwind + Headless UI | +| **9. Tests + observability** | Testcontainers: RabbitMQ, Redpanda (Kafka), LocalStack (SQS), Mosquitto (MQTT); reconcile-idempotency and election tests (kill leader, assert single owner); OpenTelemetry spans/metrics per subscription | | + +## 8. Provider priorities + +### 8.1 The leverage rule: prioritize *protocols*, not *products* + +The instinct is to write one provider per brand. Resist it — several brands are the same wire +protocol, and one protocol provider unlocks a whole column of them: + +| Build this one | And you largely cover | +|---|---| +| **Kafka** (`Confluent.Kafka`) | Apache Kafka, Redpanda, Confluent Cloud, AWS MSK, Aiven, **and Azure Event Hubs** (Kafka-compatible endpoint, Standard tier and up) | +| **AMQP 1.0** (`AMQPNetLite`) | Azure Service Bus, ActiveMQ Artemis, ActiveMQ 5.x, Solace, Qpid, and parts of IBM MQ | +| **MQTT** (`MQTTnet`) | Mosquitto, EMQX, HiveMQ, VerneMQ, AWS IoT Core, Azure IoT Hub (partially) | + +Caveat, stated honestly: a generic protocol provider gets you connect/subscribe/publish, not +the proprietary extras. Azure Service Bus sessions, scheduled messages, and dead-letter +subqueues are poorly expressed over raw AMQP 1.0. So the rule is **generic protocol provider +first, thin product-specific provider later only when a client needs the extras** — and +because `ProviderCapabilities` is declared per provider, both can coexist in the registry +without core caring. + +### 8.2 Recommended order, against actual demand + +Known demand: **external RabbitMQ** (paying client, hard requirement), **Kafka** (client using +it heavily), **Azure Event Hubs** (client, already served by a bespoke custom connector), +**Pulsar** (Tuya — capability demo), **MQTT** (capability demo). Everything else has no named +client and drops to demand-driven. + +| # | Provider | Effort | Driver | What it stresses / notes | +|---|---|---|---|---| +| **0** | **RabbitMQ (external)** | S | paying client | Already phase 2 — it is the vehicle that proves supervisor + reconcile + placement. Cheapest provider because the concepts already match Bitween's model. Do not let it slip. | +| **1** | **Kafka** | L | heaviest real load, **and** the contract stress test | Consumer groups, partitions, **offset commit instead of per-message ack**, no native DLQ, ordering per partition. If the contract survives Kafka it survives everything — which is why it must be first among the plugins regardless of demand. | +| **2** | **Azure Event Hubs** | **XS–S, or M** | existing client | *Verification task before a build task:* Event Hubs exposes a **Kafka-compatible endpoint** (Standard tier and up, SASL_SSL/PLAIN with the connection string), so provider #1 may cover this client for free. Spend a day proving it against their namespace. Fall back to a native `Azure.Messaging.EventHubs` provider only if they are on Basic tier or need checkpoint-store/Capture semantics. Either way, **mine the existing custom connector**: its config surface is a ready-made `ProviderDescriptor` and its quirks are already client-validated. | +| **3** | **MQTT** | S–M | demo, and the cheapest of the demo set | Smallest effort with the most visible payoff, because MQTT demos are inherently **two-way** — subscribe to telemetry *and* publish a command back to a device. Pair it with egress 6a (§6.3); that pairing is what makes the demo land. Capability-wise it is the first real test of `SupportsCompetingConsumers` (plain MQTT has none ⇒ forced `Placement = Leader`; MQTT 5 shared subscriptions lift it) and of `SupportsHeaders = false` on 3.1.1. | +| **4** | **Apache Pulsar** | M–L | Tuya demo | `DotPulsar` is the official .NET client and adequate. Better than I first credited it: Pulsar's subscription types — exclusive / failover / shared / key-shared — map almost exactly onto `SupportsExclusiveConsumer`, `SupportsCompetingConsumers`, and ordered-key placement, so it is arguably the **best showcase** of why the capability model exists rather than a per-broker hack. | + +**Demand-driven, no named client — build when one appears:** Azure Service Bus (or generic +AMQP 1.0, which also covers ActiveMQ Artemis and Solace), Amazon SQS (S — cheapest real +provider if an AWS client shows up), Redis Streams (S), NATS/JetStream (S–M), Google Pub/Sub +(M), IBM MQ (banking/gov value, but needs the licensed client library and a test environment). + +Two corrections to my earlier ranking, now that demand is known: **Pulsar is not a defer** — +it has a named demo target and it exercises the capability model better than most. And +**Event Hubs is not the wrong Azure target** — it has a paying client; my point was only that +Service Bus is the more common *enterprise messaging* ask, and that stands as a reason to keep +Service Bus on the demand-driven list rather than a reason to demote Event Hubs. + +### 8.3 One recommendation that costs almost nothing + +**Spike the Kafka provider before phase 0 freezes the contract** — even if you don't ship it +until tier 1. Not the full provider: just write its `ProviderDescriptor` and a throwaway +subscribe loop against Redpanda in a scratch branch. Kafka is the provider whose model differs +most from RabbitMQ (offsets, not acks; groups, not queues), so it is the one that will expose +a Rabbit-shaped assumption baked into `IDataSourceConnection`. Finding that in a two-day spike +is cheap; finding it in phase 5 means a breaking change to a published abstractions package +and every provider written against it. + +### 8.4 Worth noting: the highest-demand "providers" may not be brokers + +For an integration engine, real-world client asks are often SFTP drops, database +polling/CDC, IMAP mailboxes, and file shares — not message brokers. Those are **pull-based** +and Bitween already handles them through `ReceivingJob` + receiver adapters, which is the +right home; don't stretch `DataSource` to cover them in v1. But keep it in mind as a +deliberate boundary: if pull sources later want the same connection-management, per-node +placement, and leader election that this design gives brokers, the clean move is a sibling +`IPollingProvider` in the same registry that the supervisor schedules via Quartz instead of +subscribing to — reusing `DataSource`, placement, and cluster control unchanged. + +## 10. Node health and resource pressure + +### 10.1 What v2 already gives, and the gap + +v2 ships `SW.Bitween.Api/Resources/Ops/` — `Summary`, `Consumers`, `Queues`, `Retries`, +`DeadLetters`, `Alerts` — all thin handlers over SW.Bus's `IBusDashboardDataService`, gated on +`Permissions.Monitoring.View`, and surfaced by `QueueHealthPage.tsx`. + +**But it is queue health, not node health.** `IBusDashboardDataService` reads the RabbitMQ +**Management API**, so every number is cluster-wide broker state: queue depth, incoming/ack +rates, backpressure, dead letters. `ConsumerHealth.TotalNodes` is a *count of consumers on a +queue*, not a view of the processes. There is **no per-process telemetry anywhere today** — +nothing knows a node's memory, thread count, or how many broker connections it holds. Which is +exactly the pressure this feature introduces: every external data source is a live TCP +connection, a client object, receive buffers, and (for Kafka/Pulsar) per-partition fetch +buffers that dwarf a RabbitMQ channel. + +### 10.2 Heartbeat carries the metrics + +`ClusterHeartbeatJob` (§6, Quartz, every 15–30s) is already writing `ClusterNode.LastHeartbeatOn`. +Have it write process telemetry in the same row — no new job, no scraping, no Prometheus +dependency: + +```csharp +// on ClusterNode +public long WorkingSetBytes { get; set; } // Process.WorkingSet64 +public long GcHeapBytes { get; set; } // GC.GetTotalMemory(false) +public long GcTotalAllocatedBytes { get; set; } +public int Gen2Collections { get; set; } // growth rate is the leak signal +public double CpuPercent { get; set; } // sampled between heartbeats +public int ThreadCount { get; set; } +public int ThreadPoolQueueLength { get; set; } // >0 sustained = starvation, the real symptom +public int OpenConnections { get; set; } // data-source connections held +public int ActiveSubscriptions { get; set; } +public long? ContainerMemoryLimitBytes { get; set; } // GCMemoryInfo.TotalAvailableMemoryBytes +``` + +`ThreadPoolQueueLength` and `ContainerMemoryLimitBytes` matter most in practice. Provider client +libraries spawn their own threads and do blocking work; the first sign of an over-subscribed node +is thread-pool starvation, not memory. And a working set of 900 MB means nothing until you know +whether the cgroup limit is 1 GB or 8 GB. + +**No single metric covers both providers.** Kafka's `librdkafka` allocates *native* memory on +*native* threads, so `GcHeapBytes`, `Gen2Collections`, and `ThreadPoolQueueLength` all stay flat +while the process grows toward an OOM kill — only `WorkingSetBytes` and `CpuPercent` see it. +RabbitMQ is the mirror image: managed heap and thread-pool pressure are the signals. Collect all +of them and never treat a healthy GC heap as a healthy node. + +### 10.3 Per-connection detail + +`GET /ops/nodes` (list + metrics) and `GET /ops/nodes/{nodeId}/connections`, the latter served +from the supervisor's live in-memory map rather than the DB, since it is per-process state: + +| Field | Source | +|---|---| +| data source, endpoint, provider | supervisor map | +| state | `IMessageSubscription.State` (`Starting`/`Running`/`Degraded`/`Stopped`) | +| connected since, reconnect count, last fault | supervisor bookkeeping | +| messages in/sec, in flight, last message at | ingress pipeline counters | +| lag / backlog | `ITopologyManager` where the provider can report it (Kafka consumer lag, Rabbit queue depth); `null` when it can't — a **capability**, not a guess | + +Cross-node aggregation: fan out over the node list via `IBroadcast`, or simpler and more robust, +have each node write its own connection snapshot on heartbeat and read the union from the DB. +Prefer the DB — one query, works when a node is unreachable, and it makes "node X went dark +holding four connections" visible instead of a timeout. + +### 10.4 Guard rails, because this is the actual risk + +Visibility is necessary but not sufficient; add **budgets** so a misconfiguration can't OOM a node: + +- `BitweenOptions.MaxDataSourceConnectionsPerNode` (default ~25) and + `MaxConcurrentInboundPerNode` — the supervisor refuses to start beyond them, records a + `Degraded` reason, and raises an Ops alert instead of silently over-committing. **Weight the + budget per provider** rather than counting connections flat: a descriptor-declared + `ResourceWeight` (rabbitmq 1, kafka ~8) summed against the budget, because a Kafka handle costs + an order of magnitude more memory and several times more threads than an AMQP connection — see + [provider-plan-rabbitmq-kafka.md](provider-plan-rabbitmq-kafka.md) §2.4. +- **Per-subscription prefetch/fetch-size is mandatory in every provider descriptor.** An + unbounded Kafka `fetch.max.bytes` × partitions × topics is the realistic OOM path, and it is a + config default, not a code bug. +- Memory-pressure backpressure: when `WorkingSetBytes / ContainerMemoryLimitBytes` crosses a + threshold, the supervisor stops *accepting* (pauses subscriptions) rather than being killed — + and, if `Placement` allows, sheds to another node. +- Reuse the existing `AlertEvaluator`/Ops alert surface so node alerts appear where operators + already look, rather than in a new place. + +## 11. UI: the real challenges + +### 11.1 Decision: dedicated per-provider components, descriptor stays server-side + +v2 has two descriptor-driven form precedents — `GET /adapters/{id}/GetStartupValues` → +`AdapterConfig.tsx`, and `SettingsCatalog.All` → `SettingsPage.tsx`. My first instinct was to +render provider config generically from a `ProviderDescriptor` the same way. **Having read +`AdapterConfig.tsx`, that is the wrong call. Build a dedicated component per provider.** + +**The generic renderer's payoff is zero in this architecture.** The only reason to build a +schema-driven form engine is to decouple UI releases from provider releases — to let a provider +appear without rebuilding the frontend. But v2 compiles `ClientApp` **into `SW.Bitween.Web`**, +and providers load at startup from that same deployment (§4.1). Adding a provider is already a +rebuild of the one artifact that also contains the UI. You would be paying a permanent +abstraction tax for decoupling that the deployment model makes impossible to use. (If providers +are ever shipped to customers who can't rebuild the UI, revisit this — that is the one scenario +that flips it.) + +**The generic path has already broken down once, visibly.** `AdapterConfig.tsx` is **460 lines** +to render the *simplest possible* descriptor — flat string keys with `optional`/`default`/ +`secret`/`description` — and every field renders as **one control type, a textarea**. There are +no enums, booleans, numbers, conditional fields, or cross-field rules. And it already contains +`adapter.id === "NativeJSONMapper"` → escape to a bespoke editor: a hardcoded per-implementation +special case, which is exactly the registry pattern in its least maintainable form. + +Broker connections need all the things that descriptor can't express: SASL mechanism as an enum +that *changes which fields appear*, TLS toggle with dependent cert fields, IAM-role vs +access-key auth modes, numeric prefetch/fetch sizes with bounds, and validation that spans +fields. Extending the generic renderer to cover that means building a form framework. Five +hand-written forms are less code, better UX, and no shared abstraction to break. + +**And the primitives for hand-written forms are already complete.** `components/ui/forms.tsx` +ships `Field`, `TextInput`, `PasswordInput`, `Checkbox`, `Select`; `basics.tsx` ships `Button`, +`Badge`, `FormError`, `EmptyState`; plus `Panel`, `SearchSelect`, `KeyValueEditor`, +`SummaryDisclosure`, and a wizard kit (`WizardShell`, `StepNav`, `OptionCard`, +`usePersistentDraft`). A Kafka connection form is **composition, not new components** — a few +hundred lines of layout. + +**Keep `ProviderDescriptor` on the backend regardless.** Its job changes from *rendering +instruction* to **validation and metadata contract**, and it is still required: + +- the API must reject invalid `Settings`/`Binding` from *any* client, not just this UI (§3.1); +- `secret: true` is what routes a field through `SettingsProtector`; +- `GET /providers` powers the provider picker (labels, icons, blurbs) and a **generic fallback + form** for a provider with no dedicated component, so an unknown provider degrades to a plain + key/value editor instead of being unusable. + +**Secrets: reuse `SettingsProtector`, not `AESCryptoService`.** It is the newer design — +AES-GCM, PBKDF2 100k iterations, `enc.v1:` prefix, fresh salt+nonce per value, passphrase from +`BitweenOptions.SettingsEncryptionKey` (configuration only, never stored). Inherit its rule too: +`IsConfigured == false` ⇒ **refuse to store**. For a broker credential that means a data source +with a password cannot be saved at all on an install with no passphrase configured — surface +that as a blocking, explanatory validation error at the top of the form, not a silent drop. + +**Note what `SettingsCatalog` deliberately excludes:** its membership rule is that a setting +qualifies *only* if consumers read it from the options singleton per call — "anything captured +once during `Startup.ConfigureServices` — **the bus**, CORS, storage, JWT, Quartz, the DB +provider — stays environment-only." Broker connections are exactly that class today, which is +why `DataSource` must be its own entity with its own reconciliation (§5) rather than a Settings +row. That is the feature, stated in the codebase's own words. + +### 11.2 Test connection is the highest-value control, and it is not trivial + +`POST /data-sources/test` must accept an **unsaved draft** (otherwise you can't test before +committing bad credentials), which means: server-side timeout (~10s) so a wrong host can't hang +a request thread; a distinguishing result, not a boolean — +`{ ok, stage: dns|tcp|tls|auth|authorize|topology, elapsedMs, message, providerDetail }`, because +"auth failed" and "TLS handshake failed" send an operator to completely different places; and +**it must run on the node that will hold the connection**, or it proves nothing about a firewall +rule that only blocks node B. Route it through the supervisor on the target node (§6.2) and let +the operator pick, defaulting to the placement target. + +Secret handling in a draft test: never round-trip ciphertext to the browser. When editing an +existing source, send a sentinel (`"__unchanged__"`) and have the server substitute the stored +value. + +### 11.3 Making external gateways visible + +v2 already has `pages/bus-gateways/` (`BusGatewayPage`, `BusGatewayNewPage`, `AddRouteWizard`, +`EditRoutePage`) built for the internal-only model. Extend rather than fork: + +- **Data source column/badge** on the gateway list — provider icon + name, or "Internal bus" for + `DataSourceId == null`. An operator must never have to open a record to learn which broker it + listens to. +- **Live state chip** per gateway: `Running` / `Degraded` / `Stopped` / `Not placed here`, plus + the owning node when placement is `Leader`. Poll via TanStack Query with an interval; nothing + else in the app needs websockets, so don't introduce them for this. +- **Nodes page** under "Operate" next to Queue health: node table with the §10.2 metrics, leader + badge, per-node enable/disable, and an expandable per-connection list. Gate on + `monitoring.view`; gate the toggles on a new write permission. +- **Follow the mock-first contract.** The UI is written against a single `ApiClient` interface + with `api/mock` and `api/http` implementations — build the mock first and the whole DataSource + UI is reviewable before any endpoint exists. Genuinely worth exploiting on a feature this + large. +- **RBAC:** v2 gates every handler on `Permissions.*`. A new `data-sources` permission area + (`view` / `edit`) plus a `monitoring.manage` for node toggles must be added to the catalog and + to `nav.ts`, or the pages silently won't render for anyone. + +### 11.4 The provider UI registry + +One registry in the ClientApp, keyed by provider — the same idea as `nav.ts` being the single +source of the information architecture: + +```ts +export interface ProviderUi { + key: string; // matches IDataSourceProvider.Key + label: string; // "Apache Kafka" + icon: LucideIcon; + blurb: string; // one line for the OptionCard in the new-source wizard + ConnectionForm: FC<{ value: Settings; onChange: (s: Settings) => void; disabled: boolean }>; + BindingForm: FC<{ direction: EndpointDirection; value: Binding; onChange: (b: Binding) => void }>; + defaults: () => { settings: Settings; binding: Binding }; + summarize: (s: Settings) => string; // "3 brokers · SASL_SSL" for the list row + docsHref?: string; +} + +export const PROVIDERS: Record = { rabbitmq, kafka, eventhub, mqtt, pulsar }; +``` + +What stays **shared, written once**: the data-source page shell, the test-connection panel +(§11.2), the endpoint list, the secret field, the reference-token menu, and the wizard chrome +(`OptionCard` grid over `PROVIDERS` is the provider picker). What is **per provider**: only the +field layout inside `ConnectionForm` / `BindingForm`. Unknown key ⇒ `GenericProviderForm` driven +by the descriptor. + +Bindings are where dedicated components earn their keep hardest. "Kafka topics + consumer group + +starting offset + fetch bounds", "exchange + routing key + queue to declare + prefetch", "MQTT +topic filters + QoS + shared-subscription group", and Pulsar's four subscription types are +different *shapes*, not different fields. `MatchExpressionEditor.tsx` and `ScheduleEditor.tsx` +are the precedent: a real domain-specific editor sitting beside the generic controls. + +### 11.5 Extract three things out of `AdapterConfig.tsx` first + +Most of those 460 lines are not the form — they are reusable machinery currently trapped inside +it, and **extracting them before five provider forms exist is much cheaper than after**: + +| Extract | Why every provider form wants it | +|---|---| +| `SecretField` | mask + **Replace** button; the exact interaction a broker password needs | +| `ReferenceMenu` | searchable `{{globals.…}}` / `{{partner.…}}` token inserter, inserts at the caret | +| `ReferenceHints` | resolves the tokens in a value inline — globals to their literal, partner keys to "defined by 3 of 7 partners" with drill-down | + +**This also corrects the egress recommendation in §6.3.** I suggested Scriban templates from the +MappingEditor for dynamic binding fields. Wrong tool: adapter properties **already** support +`{{partner.KEY}}` / `{{globals.SET.KEY}}` interpolation, with UI affordances for authoring *and* +previewing them. Dynamic egress bindings (device id → MQTT topic, tenant → Kafka key) should use +that same token syntax and those same two components — one templating mechanism in the product, +not two. + +## 12. Open questions + +1. **Ordering.** Do any client integrations need per-key ordering end to end? If yes, the + ack-then-pipeline model needs a per-key serialization gate, which is a real design change + — better to know now than at phase 5. +2. **Multi-tenancy.** Should a `DataSource` be scoped to a `Partner`, or shared with the + partner chosen per gateway route (today's model)? Affects the API surface. +3. **Duplicate suppression.** `Document.DuplicateInterval` exists; does it apply to external + ingest, and keyed on what — `ProviderMessageId` or payload hash? +4. **Credentials at rest.** Is `AESCryptoService` acceptable for client broker credentials, + or do we need per-install KMS / Azure Key Vault references (`UseAzureManagedIdentity` + already hints at that direction)? +5. **Per-node addressing in SW.Bus.** Targeted control messages currently require + listener-side filtering because `NodeExchange` uses one shared routing key. Worth folding + into the same SW-Bus PR as election: add a per-node routing key + (`{NodeRoutingKey}.{NodeId}`) plus `IBroadcast.BroadcastTo(nodeId, message)` — cheap and + backward compatible, since the existing binding stays. +6. **Election scope granularity.** Per-gateway (`"gateway:{id}"`) spreads load best but means + N exclusive queues for N gateways; per-data-source is coarser but keeps one broker + connection firmly owned by one node. Recommendation: **per data source**, since a + connection is the expensive resource and gateways on the same broker should share it. + Worth confirming against expected gateway counts per install. diff --git a/docs/external-bus-providers.md b/docs/external-bus-providers.md new file mode 100644 index 00000000..886721fe --- /dev/null +++ b/docs/external-bus-providers.md @@ -0,0 +1,98 @@ +# External bus providers + +A `BusGateway` can now be fed by an external broker instead of the internal bus, through a +resident serverless adapter. + +**Nothing existing changes.** `BusGateway.DataSourceId` is nullable and null still means the +internal bus, so every gateway already in a database behaves exactly as before. The migration +(`ExternalBusDataSources`) is additive: three columns and one table. + +## The shape + +``` +external broker -> resident adapter (owns the connection, one process) + -> BusProviderEventSink (resolves the gateway, persists the Xchange) + -> XchangeService.SubmitFilterXchange + -> filter -> mapper -> handler -> auto-retry (unchanged) + -> ack returns -> adapter acknowledges its broker +``` + +Past the sink, ingress from a broker and ingress from the API are the same thing. Filtering, +mapping, work-group routing and the audit trail are not reimplemented. + +**The adapter does not acknowledge its broker until Bitween has persisted.** A rejection means +requeue, not loss — a Bitween outage stops draining the customer's queue rather than dropping +their messages. A crash between persisting and acknowledging means redelivery, which is why every +event carries a dedupe key. + +## What lives where + +| | | +|---|---| +| `DataSource` | How to reach the system: endpoint, credentials, health. One per broker. | +| `BusGateway.DataSourceId` | Which broker feeds this gateway. Null = internal bus. | +| `BusGateway.Endpoint` | Which queue or topic on it. | +| `BusGateway.EndpointProperties` | Per-subscription overrides: prefetch, visibility timeout. | +| `BusGateway.DocumentId` + routes | Unchanged — what the message *means* and what runs. | + +One data source serves many gateways, exactly as one connection serves many queues. + +## Turning it on + +```json +"Bitween": { "BusProvidersEnabled": true, "BusProviderMaxInFlight": 16 } +``` + +**Off by default, and single-instance only for now.** A broker connection is exclusive, so exactly +one node may hold it — and placement across nodes is not implemented yet, so every instance would +try. `DataSource.OwnedByNode` exists for that election to write into. Run this on one instance +until it lands. + +## The two providers + +### `SW.Bitween.Adapters.Bus.RabbitMq` + +An external RabbitMQ — someone else's broker, not Bitween's own. Consumes with `autoAck: false`, +acks only after Bitween persists, nacks with requeue on rejection. `DeclareMode` is `none`, +`assert` (passive declare, fail loudly) or `create`; `assert` is the default because silently +creating queues on a customer's broker is not our call. + +Dedupe key is the broker's message id, or a content hash — **not** the delivery tag, which is per +channel and restarts at 1 on every reconnect. + +Also supports `Publish`, so egress works on external gateways even though the internal one does +not have it yet. + +### `SW.Bitween.Adapters.Bus.Sqs` + +SQS is polled, not pushed, and has no ack — only delete. Same contract by a different mechanism: + +``` +receive -> persist -> ONLY THEN DeleteMessage +``` + +A rejection resets visibility to 0 so it retries in seconds rather than waiting out the timeout. +**`VisibilityTimeoutSeconds` must exceed how long Bitween takes to persist**, or a message is +redelivered while the first copy is still being handled — `TestConnection` warns below ~30s. + +Credentials are optional: leave them blank to use the ambient chain (instance profile, IRSA, +environment), which is the right answer on AWS. + +**Why this one matters beyond SQS itself:** the Amazon Selling Partner API delivers notifications +by publishing to an SQS queue *you* own — you create the queue, grant SP-API permission to send to +it, then subscribe notification types to that destination. So this adapter is the transport for +SP-API notifications, and `UnwrapSellingPartnerNotification` handles the envelope they arrive in, +promoting `notificationType` and the metadata into headers so a Bitween document schema does not +have to carry Amazon's wrapper. + +The SP-API *request/response* calls are ordinary HTTPS and belong in a mapper or handler, not +here. This covers the push half. + +## Not done yet + +- **Node placement and leader election** — the reason this is off by default. +- **CRUD API and UI** for `DataSource`. Rows must be inserted directly for now. +- **Secret protection at rest.** `SecretProperties` names the fields; wiring it to + `SettingsProtector` is outstanding, so treat credentials in `DataSource.Properties` as + plaintext until that lands. +- **Integration tests** against a real broker, in the style of the SW-Serverless suite. diff --git a/docs/provider-plan-rabbitmq-kafka.md b/docs/provider-plan-rabbitmq-kafka.md new file mode 100644 index 00000000..9382518d --- /dev/null +++ b/docs/provider-plan-rabbitmq-kafka.md @@ -0,0 +1,734 @@ +# Provider Plan: RabbitMQ (full) and Kafka (connection & resource shape) + +Companion to [external-brokers-architecture.md](external-brokers-architecture.md). Baseline +`origin/v2`. Numbers marked **(verify)** are library defaults to re-check against the exact +package version pinned at implementation time; they are stated because the design depends on +them, not because they should be trusted unread. + +--- + +## Part 1 — RabbitMQ provider + +### 1.1 Principle: model the broker, not SW.Bus + +`SimplyWorks.Bus` is an *opinionated application bus* built on RabbitMQ: it owns naming, invents +`.retry`/`.bad` queues, routes by .NET message-type name, and assumes one exchange per +environment. Every one of those opinions is correct for a microservice bus and wrong for a +gateway into a **client's existing broker**, where the queue is called `ORDERS.INBOUND` because +someone decided that in 2014. + +So `SW.Bitween.DataSources.RabbitMq` takes a dependency on `RabbitMQ.Client` **directly** and does +not reference `SW.Bus` at all. Concretely, what it deliberately does *not* inherit: + +| SW.Bus behaviour | Provider behaviour | +|---|---| +| queue name `{env}.{app}.{consumer}.{messageType}` | the operator types the exact name; no prefix, no lowercasing, no derivation | +| auto-declares `.retry` + `.bad` queues per consumer | declares **nothing** unless topology says so; retries are `DelayedRetry` (§2.4 of the architecture doc) | +| routes on .NET type name → one exchange per environment | routing key, exchange, and headers are configuration; any exchange type | +| `AutomaticRecoveryEnabled` left on (client-side recovery) | **off** — the supervisor owns reconnection (§1.8) | +| publish = `IPublish.Publish(name, json)` | publish = exchange + routing key + properties + confirms (§1.6) | +| one connection for the whole app | one connection per `DataSource`, one channel per subscription (§1.9) | + +### 1.2 Connection settings — mirror AMQP, plus pass-through + +Field list follows `ConnectionFactory` / the AMQP URI spec rather than anything Bitween-shaped: + +```jsonc +{ + "hosts": ["rabbit-1:5672", "rabbit-2:5672"], // ordered; client tries each (cluster-aware) + "virtualHost": "/", + "username": "bitween", "password": "…", // secret ⇒ SettingsProtector + "authMechanism": "plain", // plain | external (mTLS) | (extensible) + "tls": { + "enabled": true, "serverName": "rabbit.client.com", + "clientCertPath": null, "clientCertPassphrase": null, // secret + "acceptablePolicyErrors": [], // explicit, never a blanket "trust all" + "version": "Tls12,Tls13" + }, + "requestedHeartbeatSeconds": 60, + "requestedFrameMax": 0, // 0 = broker default (128 KiB) + "requestedChannelMax": 2047, + "connectionTimeoutMs": 30000, + "clientProvidedName": "bitween/{node}/{dataSource}", // shows in the management UI + "managementUrl": null, // optional; enables Browse + lag reporting + "clientProperties": { } // free-form pass-through +} +``` + +Two deliberate choices: + +- **`clientProvidedName` is templated and defaults to something identifying.** When a client's + DBA asks "what is this connection", the management UI must answer. Costs nothing, saves an + incident call. +- **`acceptablePolicyErrors` is an explicit list, never a boolean.** "Ignore certificate errors" + as a checkbox is how a self-signed cert quietly becomes a MITM-tolerant production connection. + +### 1.3 Topology — mirror `definitions.json`, support everything + +The topology model is a subset of RabbitMQ's own export format, so an operator can paste from +`rabbitmqadmin export` or the management UI's definitions file: + +```jsonc +{ + "exchanges": [ + { "name": "orders", "type": "topic", "durable": true, "autoDelete": false, + "internal": false, "arguments": { "alternate-exchange": "orders.unrouted" } } + ], + "queues": [ + { "name": "ORDERS.INBOUND", "durable": true, "exclusive": false, "autoDelete": false, + "arguments": { + "x-queue-type": "quorum", // classic | quorum | stream — all allowed + "x-dead-letter-exchange": "orders.dlx", + "x-dead-letter-routing-key": "inbound.failed", + "x-max-length": 100000, "x-overflow": "reject-publish", + "x-message-ttl": 86400000, "x-max-priority": 10, + "x-single-active-consumer": true, + "x-quorum-initial-group-size": 3 + } } + ], + "bindings": [ + { "source": "orders", "destination": "ORDERS.INBOUND", "destinationType": "queue", + "routingKey": "order.created.*", "arguments": {} }, + { "source": "orders", "destination": "orders.audit", "destinationType": "exchange", + "routingKey": "#", "arguments": {} } + ] +} +``` + +Non-negotiables for "not opinionated": + +- **All exchange types, including plugin ones.** `direct`, `fanout`, `topic`, `headers`, plus + `x-consistent-hash`, `x-delayed-message`, `x-random`, `x-modulus-hash` — the type is a + **free-text string**, not an enum, because a plugin can define one we've never heard of. Validate + by attempting the declare and surfacing the broker's error, not by rejecting unknown strings. +- **`arguments` is an open map**, passed through verbatim with correct AMQP type coercion + (integers as `long`, `x-match` as string, nested tables). Never a fixed field list — the + `x-*` argument space grows every RabbitMQ release. +- **Exchange-to-exchange bindings** (`destinationType: "exchange"`) are first class. Every + hand-rolled integration forgets these and then can't model a real client topology. +- **Headers exchanges** need `x-match: all|any` plus arbitrary header keys in binding arguments — + which falls out of the open-map rule. + +### 1.4 Declare mode — the field that decides whether this works at clients + +``` +"declareMode": "none" | "assert" | "create" +``` + +| Mode | Behaviour | When | +|---|---|---| +| `none` | touch nothing; just consume/publish | Bitween has only `read`/`write` on the vhost — **the common case at a client** | +| `assert` | `QueueDeclarePassive` / `ExchangeDeclarePassive`; fail fast with a clear error if missing | verify the client actually created what they promised, without needing `configure` | +| `create` | full declare + bind, idempotent | Bitween owns the topology | + +This single field is what makes the provider usable against brokers we don't administer. A +provider that always declares will fail with `ACCESS_REFUSED` at exactly the wrong moment — and +worse, a declare with mismatched arguments returns `PRECONDITION_FAILED` **and kills the +channel**, which is an easy way to look broken while being correct. + +### 1.5 Inbound binding + +```jsonc +{ + "queue": "ORDERS.INBOUND", + "consumerTag": "", // "" ⇒ broker-generated + "prefetchCount": 20, // REQUIRED (§1.10) — the memory bound + "prefetchSize": 0, + "exclusive": false, // exclusive consumer (≠ exclusive queue) + "arguments": { "x-priority": 5, "x-stream-offset": "last" }, + "noAck": false, // exposed, defaulted false, warned in UI + "contentEncoding": null // optional override when the publisher lies +} +``` + +Notes that matter: **prefetch is per-channel**, so one channel per subscription is required to +give each its own prefetch (§1.9). `x-stream-offset` is how stream queues are consumed — +supported by virtue of `arguments` being open. `noAck: true` is exposed because some telemetry +feeds genuinely want it, but it breaks the persist-then-ack guarantee, so the UI must say so. + +### 1.6 Outbound binding + +```jsonc +{ + "exchange": "orders", // "" ⇒ default exchange (publish straight to a queue) + "routingKey": "order.created.{{partner.region}}", // token-templated (§11.5) + "mandatory": false, // true ⇒ surface basic.return as a publish failure + "confirms": true, // publisher confirms; wait per-publish or per-batch + "confirmTimeoutMs": 5000, + "properties": { + "deliveryMode": 2, // 1 transient | 2 persistent + "priority": null, "expiration": null, "contentType": "application/json", + "headers": { "x-tenant": "{{partner.code}}" } + } +} +``` + +`mandatory` + `confirms` together are the only way to know a publish landed. Without them a +publish to a nonexistent routing key succeeds silently — a failure mode that looks exactly like +success in every log. Default `confirms: true` for egress, and treat a nack or a +`basic.return` as a transport failure feeding the outbox retry, not an `Xchange` error. + +### 1.7 Capabilities + +```csharp +new ProviderCapabilities( + CanManageTopology: true, CanBrowseTopology: true, // Browse requires managementUrl + SupportsHeaders: true, SupportsOrderingKey: false, // routing key ≠ ordering key + SupportsPartitions: false, + SupportsNack: true, SupportsRequeueDelay: false, // no native delay without the plugin + SupportsNativeDeadLetter: true, // x-dead-letter-exchange + SupportsExclusiveConsumer: true, // exclusive consumer / x-single-active-consumer + SupportsCompetingConsumers: true, + SupportsTransactions: false, // tx.* is slow; use confirms instead + SupportsBatch: true, MaxMessageBytes: 134_217_728); +``` + +`SupportsRequeueDelay` becomes `true` when the `rabbitmq_delayed_message_exchange` plugin is +detected — SW.Bus already probes for it (`DelayedPluginAvailable`), so reuse the technique: +probe once at connect, cache on the connection, report through capabilities. Capabilities are +therefore **per connection**, not per provider type — worth making the contract return them from +`IDataSourceConnection`, not only `IDataSourceProvider.Describe()`. + +### 1.8 Reconnection: the supervisor owns it + +Set `AutomaticRecoveryEnabled = false` and `TopologyRecoveryEnabled = false`. Reasons: + +1. Two recovery mechanisms fight. The supervisor already reconciles on fault (§5), and + client-side recovery silently re-declares topology that `declareMode: none` says not to touch. +2. Client-side recovery hides state: `IMessageSubscription.State` and reconnect counts (§10.3) + would be lies. +3. Recovery policy belongs with placement — on a `Leader`-placed subscription, a dropped + connection may mean *another node should take over*, not that this one should reconnect. + +Instead: `ConnectionShutdown` / `CallbackException` / consumer `Shutdown` → mark `Degraded`, +raise `Faulted`, let the supervisor back off (exponential, jittered, capped) and reconcile. + +### 1.9 Connection and channel topology inside the provider + +- **One `IConnection` per `DataSource`.** Not per subscription — a connection is the expensive + object; channels are cheap. +- **One `IModel`/`IChannel` per subscription.** Required for per-subscription prefetch, and + channels are *not* thread-safe, so sharing one across consumers is a data race waiting for load. +- **One dedicated publish channel per (connection, endpoint) for egress**, because + `confirm_select` is a channel-level mode and mixing confirmed publishes with consumer acks on + one channel makes the confirm bookkeeping ambiguous. +- Never publish from a consumer's channel. + +### 1.10 Resource footprint — RabbitMQ + +**The dominant term is unacknowledged messages, and it is entirely under our control:** + +``` +peak inbound memory ≈ Σ over subscriptions ( prefetchCount × avg message size ) +``` + +A prefetch of 500 on a queue of 2 MB payloads is 1 GB of managed memory on one node, from one +config field. Hence: **`prefetchCount` is a required field in the descriptor with a sane default +(10–20) and an explicit upper bound**, and the UI shows the computed worst case +(`prefetch × observed **max** size`) next to it. That single affordance prevents most of the +plausible OOMs. + +Calibrate against the measured baseline in §5.2 of the architecture doc: real payloads run ~5 KB +average with a ~1 MB maximum, and ingest acks in ~330 ms. At those sizes prefetch is cheap +(20 × 5 KB = 100 KB) and the bound should be set by the **maximum** payload, not the average — +prefetch 500 × 1 MB is 500 MB, prefetch 20 × 1 MB is 20 MB. Don't be timid with prefetch for +ingest; the downstream work-group queue is the shock absorber, not the broker. + +Per-connection and per-channel costs (**verify** against the pinned version): + +| Item | Rough cost | Notes | +|---|---|---| +| `IConnection` | tens of KB managed + socket buffers; frame buffers scale with negotiated `frame_max` (128 KiB default) | broker side also pays ~100 KB+ per connection | +| TLS | + `SslStream` buffers, order of 32–64 KB per connection | plus handshake CPU at connect/reconnect | +| `IModel` per subscription | small — hundreds of bytes to low KB | not a scaling concern | +| **Threads (client 6.8.1)** | **one dedicated socket read loop per connection**, plus a heartbeat timer; consumer callbacks dispatched on the async work service | 25 data sources ⇒ ~25 dedicated threads before any work happens | + +CPU is negligible at rest — heartbeats every 60 s and frame parsing proportional to throughput. +The two real CPU events are TLS handshakes during reconnect storms (bounded by the backoff in +§1.8) and JSON deserialization in Bitween's own pipeline, which dwarfs the AMQP cost. + +**Client version decision.** SW.Bus pins `RabbitMQ.Client 6.8.1` (`IModel`, +`DispatchConsumersAsync`). The provider loads in its **own `AssemblyLoadContext`** (§4.1), so it +can independently target **`RabbitMQ.Client` 7.x** — `IChannel`, fully async, task-based I/O +instead of a thread per connection, and a better allocation profile. Two managed versions +coexisting in one process is exactly what the ALC isolation buys, and there is no native +dependency to conflict. **Recommendation: build the provider on 7.x** and treat the thread +savings as the headline reason. If 7.x proves troublesome, 6.8.1 works — but then budget one +thread per data source and lower `MaxDataSourceConnectionsPerNode` accordingly. + +### 1.11 Test-connection stages + +Map to the staged result from §11.2 — each stage is a distinct RabbitMQ failure the operator +fixes differently: + +| Stage | Check | +|---|---| +| `dns` / `tcp` | resolve + connect to each host in `hosts`, report which succeeded | +| `tls` | handshake; report cert subject, expiry, and the specific policy error | +| `auth` | `CreateConnection` — distinguish `ACCESS_REFUSED` (credentials) from vhost-not-found | +| `authorize` | probe the three permissions separately: `read` (passive-declare the queue), `write` (publish to a nonexistent routing key on a topic exchange with `mandatory:false`), `configure` (declare a temporary auto-delete queue, then delete) — and report which are held. This is what tells an operator that `declareMode: create` will fail *before* they save. | +| `topology` | `assert`/`create` dry run: passive-declare everything the endpoint references | + +Also report negotiated `frame_max`, `channel_max`, server version, and detected plugins +(delayed message, consistent hash) — all cheap, all useful, all invisible otherwise. + +### 1.12 Cluster discovery via the management plugin + +**Difficulty: low, and the payoff is the best demo surface in the whole feature.** The management +plugin is a plain HTTP+JSON API (`:15672`, `:15671` for TLS) with HTTP basic auth. A typed client +over the eight endpoints below is roughly a day's work; the interesting design is entirely in +degradation and scale, not in the calls. + +Endpoints that carry the whole experience (**verify** shapes against the cluster's version — the +management API is stable but has grown fields across 3.8 → 4.x): + +| Endpoint | What it unlocks | +|---|---| +| `GET /api/overview` | cluster name, RabbitMQ + Erlang version, **`exchange_types`** (see below), listeners, rates mode | +| `GET /api/nodes` | per-node memory/disk alarms, fd/socket usage, partition status — "is this cluster healthy" | +| `GET /api/vhosts` | vhost picker instead of a free-text field | +| `GET /api/exchanges/{vhost}` · `GET /api/queues/{vhost}` | the browsable topology, with live depth/rates per queue | +| `GET /api/bindings/{vhost}` | the routing graph (and the routing preview, below) | +| `GET /api/whoami` + `GET /api/permissions/{vhost}/{user}` | the configure/write/read regex triple for *this* user | +| `GET /api/definitions/{vhost}` | the **entire topology in the exact shape §1.3 already models** | +| `GET /api/aliveness-test/{vhost}` | broker-side end-to-end check for the health panel | + +#### The five features worth building + +1. **Build the exchange-type dropdown from the live cluster.** `/api/overview` returns + `exchange_types` — what *this* broker actually supports, including plugin types like + `x-consistent-hash` and `x-delayed-message`. This is the perfect answer to "don't be + opinionated": no hardcoded list, no guessing about plugins, and it self-updates when the + client installs a plugin. Same trick replaces the separate delayed-plugin probe in §1.7. + +2. **Import instead of retype.** `GET /api/definitions/{vhost}` returns exchanges, queues, and + bindings in the structure §1.3 deliberately mirrors — so "select these three queues and two + exchanges → save as this endpoint's topology" is a filter over a JSON document, not a + translation layer. This is the single biggest reason the definitions-shaped topology model was + the right choice. + +3. **Drift and conflict detection — the highest-value item.** With live definitions, the + `TopologyDiff` from `ITopologyManager.Plan` stops being a guess: show *exactly* what `Apply` + would create, what already matches, and — critically — **what exists with different + arguments**. Argument mismatch is the failure that returns `PRECONDITION_FAILED` and **kills + the channel** (§1.4); catching it in a diff before saving turns the nastiest RabbitMQ failure + mode into a UI warning. + +4. **Preflight warnings computed from real numbers, not guesses.** `GET /api/queues/{vhost}/{name}` + returns `messages`, `message_bytes`, `consumers`, `consumer_details`, `consumer_utilisation`, + and rates. Two warnings fall straight out: + - **Memory:** average message size is `message_bytes / messages`, so the prefetch estimate in + §1.10 becomes `prefetchCount × actual observed average` instead of a hand-waved number. + - **Competing consumers:** if the queue already has consumers, say so plainly — Bitween will + compete with them for messages, or be silently starved if the queue has + `x-single-active-consumer`. This is a real trap that is invisible without discovery, and + "why does Bitween only get half the messages" is otherwise a day of debugging. + +5. **Routing preview, computed locally.** There is no route-simulation endpoint, but with the + bindings in hand it is ~40 lines: for `direct` compare routing keys, for `topic` match `*`/`#`, + for `fanout` take all. Then the outbound form can say *"routing key `order.created.jo` reaches + `ORDERS.INBOUND`, `ORDERS.AUDIT`"* — or, far more usefully, **"reaches no queue; this message + will be silently discarded"**, which is exactly the failure `mandatory`/confirms exists to + catch (§1.6). Be honest about the limit: `headers`, `x-consistent-hash`, and other plugin + exchanges can't be simulated — show "cannot preview for this exchange type" rather than a + wrong answer. + +#### Optional, high value, needs a policy decision + +`POST /api/queues/{vhost}/{name}/get` peeks messages without consuming (with +`ackmode=reject_requeue_true`). That would let the Information-type screen **seed promoted +properties from a real message** — a genuinely great onboarding flow. But it reads the client's +production payloads into Bitween's UI, so it needs: an explicit permission, an audit entry, a +`count=1` cap, requeue mode forced, and never running automatically on page load. Worth building; +worth not building casually. + +#### What makes it hard (none of it is the code) + +1. **Management access is a different grant from AMQP access.** The HTTP API requires a user with + a management *tag* (`monitoring` / `management` / `administrator`); AMQP `read`/`write`/ + `configure` permissions grant nothing there. Many clients will hand over AMQP credentials and + no management user at all. So: `managementUrl` stays **optional** (§1.2), + `CanBrowseTopology` is a **per-connection** capability, and every discovery feature degrades to + the AMQP-only path — you can't *list*, but `declareMode: assert` can still *verify* a name via + passive declare. Design the UI so discovery is an accelerator, never a prerequisite. +2. **The HTTP port is a separate firewall hole.** 5672 open does not imply 15672 open. Report this + as its own test-connection stage so "discovery unavailable" is distinguishable from "wrong + password". +3. **Scale.** `GET /api/queues` on a cluster with thousands of queues returns megabytes and costs + the broker real work; `/api/bindings` is worse. Use the list endpoints' pagination and column + projection (`page`, `page_size`, `name`, `use_regex`, `columns=`) from the first commit, never + fetch-all-then-filter-in-JS, and always scope by vhost. Also honour the user's `read` regex + when presenting results, so the picker doesn't offer queues they can't consume. +4. **Stats can be absent.** Metrics collection can be disabled or lag on a busy cluster; treat + depth/rate fields as nullable and show "unavailable" instead of `0`, which reads as "empty + queue" and is a much worse lie. +5. **Where it runs.** Discovery must execute on a node that can reach the management port — same + argument as test-connection in §11.2 of the architecture doc. Route it through the supervisor, + not the API node that happens to serve the request. + +#### Sizing + +| Piece | Effort | +|---|---| +| Typed management client (8 endpoints, pagination, auth, timeouts) | ~1 day | +| Browser UI: vhost → exchanges/queues tree, live columns, "use as endpoint" | 2–3 days | +| Definitions import + diff/drift panel | 1–2 days | +| Routing preview + preflight warnings | ~1 day | +| Message peek (incl. permission + audit) | ~1 day | + +Roughly **a week and a half on top of the provider**, entirely additive — every piece can ship +after the provider works, and the provider is fully functional with none of it. Note the v2 UI has +no graph/diagram library (only `lucide-react` icons), so render the binding graph as an indented +list or a small hand-rolled SVG rather than adding a dependency for it. + +### 1.13 Build order + +| Step | Deliverable | +|---|---| +| 1 | Project `SW.Bitween.DataSources.RabbitMq`, `IDataSourceProvider` + descriptor (connection fields, topology schema, inbound/outbound binding), no I/O | +| 2 | `Connect` + staged health check (§1.11) + capability probe (plugins, server version) | +| 3 | `Subscribe`: channel per subscription, prefetch, `AsyncEventingBasicConsumer` (or 7.x async consumer) → `InboundMessage`; ack/nack from `AckDecision`; `ProviderMetadata` carries deliveryTag, exchange, routingKey, redelivered, headers | +| 4 | Fault plumbing: shutdown/callback events → `Faulted` + `State`, no client auto-recovery | +| 5 | `ITopologyManager`: `Plan` (diff against live definitions when the management API is available, passive declares otherwise), `Apply` (idempotent), `Browse` (management API when configured) | +| 6 | `Publish`: dedicated channel, confirms, `mandatory` + `basic.return`, templated routing key and headers | +| 7 | Integration tests (Testcontainers RabbitMQ, with and without the delayed plugin — SW.Bus's `PluginAvailableTests`/`PluginUnavailableTests` are the pattern): all four core exchange types + consistent-hash, e2e binding, quorum and stream queues, `declareMode` × permission matrix, `PRECONDITION_FAILED` on argument mismatch, kill-the-broker reconnect, prefetch honoured | + +--- + +## Part 2 — Kafka provider: connection and resource shape + +Deliberately scoped to what the user asked: what is hard about *connecting*, and what it costs in +memory and CPU. Semantics (offsets, groups, ordering) are covered in the architecture doc. + +### 2.1 Configuration: pass-through is the correct answer + +Do **not** model librdkafka's ~200 properties as fields. First-class the handful Bitween's +behaviour depends on, and expose the rest as a validated pass-through map: + +```jsonc +{ + "bootstrapServers": "b1:9092,b2:9092", + "securityProtocol": "sasl_ssl", // plaintext | ssl | sasl_plaintext | sasl_ssl + "saslMechanism": "scram-sha-512", // plain | scram-sha-256/512 | gssapi | oauthbearer + "saslUsername": "…", "saslPassword": "…", // secret + "ssl": { "caLocation": null, "certificateLocation": null, "keyLocation": null, + "keyPassword": null, "endpointIdentificationAlgorithm": "https" }, + "clientId": "bitween-{node}", + "config": { "socket.keepalive.enable": "true", "…": "…" } // free-form librdkafka pass-through +} +``` + +Guard the pass-through with a **deny list**, not an allow list: reject keys that would break the +ack model (`enable.auto.commit` — Bitween commits after persistence) and keys the provider owns +(`group.id`, `bootstrap.servers`, anything in the first-class set). Everything else goes through, +because the next client will need `sasl.oauthbearer.token.endpoint.url` or a broker-version +override and should not need a Bitween release to get it. + +Event Hubs note: its Kafka endpoint is `sasl_ssl` + `PLAIN` with username `$ConnectionString` +and the connection string as password. That is a *configuration* of this provider, which is the +whole point of the §8.2 sequencing — try it before writing an Event Hubs provider. + +### 2.2 Connection challenges + +1. **A "connection" is not a connection.** `Confluent.Kafka` wraps native **librdkafka**; a + consumer handle opens a socket to *every* broker it learns about, not just the bootstrap. Your + connection count is a function of the client's cluster size, which you don't control. +2. **Failures are asynchronous and log-shaped.** There is no `Connect()` to await. A wrong + password surfaces as an error *event* (`ErrorCode.SaslAuthenticationFailed`) some time after + construction. So test-connection must be implemented as **`AdminClient.GetMetadata(timeout)`** + with an error-handler subscription, not as "did the constructor throw". Budget a real timeout + (10 s) and treat "no metadata yet" as failure with the collected error text. +3. **Handle disposal is slow.** Closing a consumer performs a leave-group round trip and can take + seconds. The supervisor's reconcile must dispose off the critical path, or a config save + appears to hang. +4. **Rebalances make reconcile expensive** — and this is the real interaction with the cluster + design. Every stop/start of a Kafka subscription triggers a consumer-group rebalance that + pauses *all* members. A flapping node or an over-eager reconcile becomes a rebalance storm. + Mitigations, all cheap, all needed: + - **debounce reconcile per data source** (e.g. coalesce for 5–10 s) instead of acting on every + broadcast; + - set **`group.instance.id`** from the stable `NodeId` (static group membership) so a restart + inside `session.timeout.ms` does **not** rebalance; + - prefer `partition.assignment.strategy=cooperative-sticky` so a join doesn't stop the world; + - **`Placement = All` is the right default for Kafka** — the consumer group already does the + distribution, and forcing `Leader` both wastes nodes and makes every leadership change a + rebalance. +5. **One handle per (data source, group), not per topic.** A single consumer can subscribe to + many topics; a handle per topic multiplies threads and sockets for nothing. + +### 2.3 Memory — the headline risk + +librdkafka prefetches **per partition**, and the defaults are large (**verify** against the pinned +version): + +| Property | Default | Meaning | +|---|---|---| +| `queued.max.messages.kbytes` | 65536 (**64 MiB**) | **per partition** local queue cap | +| `queued.min.messages` | 100000 | messages to keep queued per partition | +| `fetch.message.max.bytes` | 1048576 (1 MiB) | per-partition fetch request size | +| `fetch.max.bytes` | 52428800 (50 MiB) | per fetch response, across partitions | +| `receive.message.max.bytes` | 100000000 | hard per-response ceiling | + +Worst case on defaults: **partitions × 64 MiB**. A 30-partition topic is ~2 GB of prefetch buffer +for one subscription — on a node whose container limit might be 1 GB. This is not a hypothetical; +it is the default configuration. + +So: **`queued.max.messages.kbytes` and `queued.min.messages` are required descriptor fields with +low defaults** (start at 1024–4096 KiB and 1000), and the UI must show +`partitions × queued.max.messages.kbytes` as the reserved worst case. The provider should query +partition count at subscribe time and refuse — or loudly warn — when the product exceeds a +configured share of the node's `ContainerMemoryLimitBytes` (§10.4). + +**And it is native memory.** librdkafka allocates outside the .NET heap, so +`GC.GetTotalMemory()` and Gen2 counts show nothing while the process grows. +`Process.WorkingSet64` is the only metric in §10.2 that catches Kafka pressure — worth calling +out explicitly on the Nodes page, or an operator will conclude the node is healthy right up to +the OOM kill. + +### 2.4 CPU and threads + +- **Threads per consumer handle:** 1 main/coordinator thread + **1 per broker connection** + + internal timers. A 3-broker cluster ⇒ roughly 5 native threads *per handle*. Ten Kafka + subscriptions sharing one handle: still ~5. Ten separate handles: ~50. This is the argument for + §2.2 item 5, in numbers. +- **These are native threads, not thread-pool threads.** Decompression (lz4/snappy/zstd) and CRC + validation happen on them, so Kafka CPU load is **invisible to `ThreadPoolQueueLength`** — + the one §10.2 metric that catches RabbitMQ-side starvation. Process CPU% is the signal for + Kafka nodes. Both metrics are needed; neither is sufficient alone. +- Compression is the main steady-state CPU cost. `zstd` on a busy topic is materially more + expensive than `lz4`; expose `compression.codec` for egress and default to `lz4`. +- Practical budget: assume a Kafka data source costs roughly **an order of magnitude more + memory and several times more threads** than a RabbitMQ one. `MaxDataSourceConnectionsPerNode` + (§10.4) should therefore be **weighted per provider**, not a flat count — a descriptor-declared + `ResourceWeight` (rabbitmq: 1, kafka: 8, pulsar: 6) that the supervisor sums against a budget. + +### 2.5 Discovery, the Kafka equivalent — different in kind + +Since Kafka is the less familiar one, the plain-language version: **Kafka has no management plugin +and needs none.** The equivalent of RabbitMQ's HTTP admin API is the **Admin API carried over the +same binary protocol on the same port with the same credentials** (`AdminClient` in +`Confluent.Kafka`). That is strictly *easier* than RabbitMQ: no second port, no second firewall +hole, no separate management user, no pagination. + +What you can discover, and what it maps to: + +| Kafka call | Gives you | RabbitMQ analogue | +|---|---|---| +| `GetMetadata` | brokers, cluster id, **topics with partition counts**, replicas, leaders | `/api/overview` + `/api/queues` | +| `DescribeConfigs` | per-topic config: retention, `cleanup.policy`, `min.insync.replicas` | queue `arguments` | +| `ListConsumerGroups` / `DescribeConsumerGroups` | existing groups, their members and assignments | queue `consumer_details` | +| `ListConsumerGroupOffsets` + `QueryWatermarkOffsets` | committed offset vs high watermark ⇒ **lag** | queue depth | +| `DescribeCluster` | controller, broker endpoints | `/api/nodes` | +| `DescribeAcls` | permissions, when the user may read them | `/api/permissions` | + +**The structural difference to be clear about: there is nothing to browse.** Kafka has no +exchanges, no bindings, and no routing — a topic is a flat log, and "routing" is just the +producer's choice of topic plus a partition key. So the binding graph, the routing preview, and +the definitions import have **no Kafka analogue at all**. The equivalent smart view is a flatter +thing: topics × partitions × lag × retention, plus which consumer groups already read each topic. + +Two Kafka-specific warnings that are worth as much as the RabbitMQ ones: + +1. **Group-id collision.** In RabbitMQ, an existing consumer on the queue means Bitween *competes* + for messages. The Kafka mirror is sharper: consumer groups are independent, so reading a topic + another group already reads is harmless — but reusing a **`group.id` someone else is using** + silently steals their partitions. `ListConsumerGroups` makes this checkable at configuration + time, and it should be a hard validation warning. +2. **Topic auto-creation.** If the broker allows it and `allow.auto.create.topics` is true, a typo + in a topic name silently *creates* an empty topic and the subscription sits there consuming + nothing forever. Default that setting to **false** in the provider and validate the topic + exists via metadata instead. + +Effort: **~2 days** for the discovery client plus a topics/groups/lag panel — less work than +RabbitMQ's, with less to show, because there is genuinely less structure in Kafka to reveal. + +### 2.6 Minimal build order (see also Part 3 for the test environment) + +| Step | Deliverable | +|---|---| +| 1 | Descriptor + config validation (first-class fields, deny-listed pass-through), no I/O | +| 2 | `AdminClient.GetMetadata` staged health check with error-event capture; report cluster id, broker count, topic/partition counts | +| 3 | Consumer handle per (data source, group); subscribe many topics; manual commit after persistence; `ProviderMetadata` = topic, partition, offset, timestamp, headers | +| 4 | Resource guards: required queue-size fields, partition-count × buffer preflight against the node budget, `ResourceWeight` | +| 5 | Reconcile hygiene: debounce, `group.instance.id` from `NodeId`, cooperative-sticky, async disposal | +| 6 | Lag reporting for §10.3 via committed vs high-watermark | +| 7 | Producer for egress: `lz4`, `acks=all`, `enable.idempotence=true`, delivery-report callbacks feeding the outbox | +| 8 | Integration tests on Redpanda (fast, Kafka-protocol compatible): consume/commit/restart-resume, rebalance on second node, buffer cap honoured, Event Hubs config path validated against a real namespace | + +--- + +## Part 3 — Local dev and integration testing + +### 3.1 The actual CI situation + +CI is **GitHub Actions** (`.github/workflows/bitween-api-cicd-gateway.yml`), delegating to the +org-wide reusable workflow `simplify9/.github/.github/workflows/reusable-service-cicd.yml@main`. +`azure-pipelines.yml` is not the active pipeline. The repo is **public**, so GitHub-hosted +`ubuntu-latest` gives **4 vCPU / 16 GB RAM / 14 GB SSD** — roughly double a private-repo runner, +and generous for this workload. Resource pressure is therefore *not* the deciding factor; +**startup seconds are**, because they are paid on every run. + +**Two findings that matter more than any container sizing:** + +1. **Integration tests do not run in CI today.** The workflow passes + `test-projects: 'SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj'` — `SW.Bitween.IntegrationTests` + is never executed by any pipeline. Every container-cost estimate below is therefore + *hypothetical* until someone decides to run them. +2. **There is no PR gate.** Triggers are `push` on `releases/**` plus `workflow_dispatch`, so + pushes to `v2` — the branch this feature is being built on — run nothing at all. + +So the real decision is not "which Kafka image" but **"do we add a PR workflow that runs the +integration suite?"** Recommendation: add a separate `pr-tests.yml` on `pull_request` + +`push: [v2]` running unit **and** integration tests. Without it, every broker provider ships with +tests that only ever run on a developer's laptop — which for a feature whose whole risk surface is +live connections and reconnection behaviour is the wrong trade. The reusable release workflow +should stay as-is; this is an additive, low-risk second workflow. + +Today's suite (`SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs:33-34`) starts +`PostgreSqlBuilder` + `RabbitMqBuilder` once per collection (`[CollectionDefinition("Bitween")]`), +so the pattern for adding brokers already exists and already scales the right way. + +Side note, relevant to §1.12: the deploy step already injects +`Bitween__RabbitMqManagementUrl` / `Username` / `Password` as secrets, so **a management-API +connection is already a first-class configured dependency in production** (it backs the `Ops` +queue-health pages). The `DataSource` design simply moves that from one per install to one per +connection. + +### 3.2 Is Kafka dockerized? Yes — and there are lighter protocol-compatible options + +| Option | Image | RSS (approx) | Cold start | Verdict | +|---|---|---|---|---| +| **Redpanda** (C++, no JVM, no ZK) | `redpandadata/redpanda` | **~150–250 MB** tuned | **~3–5 s** | **Recommended for CI.** Kafka-protocol compatible, single binary | +| Apache Kafka, KRaft mode (JVM) | `apache/kafka` | ~600–800 MB with `-Xmx512m` | ~15–25 s | Most faithful reference. Keep as an opt-in "conformance" job | +| Apache Kafka native image | `apache/kafka-native` | ~150–250 MB | ~1–3 s | Very promising — GraalVM build, dev/test targeted. **(verify maturity for the version you pin)** | +| Confluent | `confluentinc/cp-kafka` / `confluent-local` | ~800 MB–1 GB | ~20–30 s | Heaviest; no advantage here | +| Azure Event Hubs emulator | `mcr.microsoft.com/azure-messaging/eventhubs-emulator` | ~1–1.5 GB (**needs an Azure SQL Edge sidecar**) | ~30 s+ | **Don't use for CI.** Two containers, and **Kafka-endpoint support is a documented limitation** (verify against the current release) | + +**Kafka no longer needs ZooKeeper.** KRaft mode has been production-ready since 3.3, so any modern +image is a single container — the two-container Kafka+ZK setup people remember is obsolete. + +**Redpanda needs explicit flags or it will eat the agent.** It is thread-per-core by design and +defaults to claiming all cores and a large memory reservation, which even on a 4-vCPU runner +starves the test host. Non-negotiable CI flags: + +``` +--overprovisioned --smp 1 --memory 512M --reserve-memory 0M --node-id 0 --check=false +``` + +`Testcontainers.Redpanda` sets sane defaults, but verify these are applied — this single item is +the difference between a 4-second start and a pegged agent. + +**Event Hubs strategy (consistent with §8.2 of the architecture doc):** test the *protocol* against +Redpanda, and validate the Event Hubs **configuration path** once against a real namespace as a +manual/nightly step. Do not try to emulate Event Hubs in the PR suite. + +### 3.3 How the community handles this + +**Testcontainers is the answer, and it is what you already do.** Java, .NET, Go, and Node all +converge on it; the .NET modules `Testcontainers.Redpanda` and `Testcontainers.Kafka` exist +alongside the `PostgreSql`/`RabbitMq` ones already in use (bump all Testcontainers packages +together — the repo is on 3.10.0). + +Worth knowing: **.NET has no embedded Kafka.** Java has `spring-kafka-test` / +`EmbeddedKafkaCluster` for in-process brokers; there is no equivalent for .NET, so a container is +the only honest option. That is a real difference from RabbitMQ testing, where SW.Bus's existing +suite already proves the container path is fine. + +The four practices that keep this cheap — and the traps they avoid: + +1. **One broker for the whole suite, not per test class.** This is *the* mistake: a broker per + fixture turns a 2-minute suite into 20. The existing collection fixture already does it right; + keep new brokers in the same fixture. +2. **Isolate with unique names, not fresh containers.** Per-test `topic-{guid}` / `group-{guid}` + (and per-test queue names for RabbitMQ) gives full isolation at zero startup cost. Never + restart a broker to get a clean slate. +3. **Pre-create topics via `AdminClient`; never rely on auto-create.** Auto-create hides the exact + typo bug §2.5 warns about, and produces flaky "consumer sees nothing" failures. +4. **`.WithReuse(true)` for the local dev loop** so an inner loop doesn't pay startup on every + run. Leave it off in CI, where a clean agent is the point. + +### 3.4 CI budget + +| Suite | Containers | RAM | Added cold start | +|---|---|---|---| +| Today | Postgres + RabbitMQ | ~200–350 MB | ~10–20 s | +| + external RabbitMQ provider | reuse the same RabbitMQ — **but see §3.5** | +0 | +0 | +| + Kafka provider (Redpanda tuned) | +1 | ~+200 MB | ~+5 s | +| + MQTT (`eclipse-mosquitto`) | +1 | **~+15 MB** | **<1 s** | +| + Pulsar standalone | +1 | **~+1–2 GB** | **~+30–60 s** | + +Everything except Pulsar fits the 16 GB / 4-vCPU runner easily; the full set including Redpanda +and Mosquitto lands around 600 MB and adds well under 10 seconds of startup. + +**Pulsar is the outlier and needs a planning decision now**, because it is tier-1 for the Tuya +demo (§8.2). Pulsar standalone is a JVM cluster in one container — BookKeeper plus broker plus +ZooKeeper-equivalent — and on a shared agent its startup alone can exceed the current entire +suite. Recommendation: **put Pulsar (and later IBM MQ) tests in a separate opt-in pipeline job** +that runs nightly or on-label, not in the PR suite. Keep the PR suite at +Postgres + RabbitMQ + Redpanda + Mosquitto. + +### 3.5 Two concrete gotchas in the current setup + +1. **The RabbitMQ discovery feature needs a different image — and SW-Bus already shows how.** + There is **nothing to install and no plugin-enabling step to automate**: the official + `rabbitmq:*-management` image variants ship with `rabbitmq_management` **pre-enabled**. It is a + one-line image override, not a provisioning problem. SW-Bus's own integration tests already do + exactly this: + + | SW-Bus test | Image | What it proves | + |---|---|---| + | `PluginUnavailableTests.cs:19` | `rabbitmq:3.13-management` | official `-management` variant = management API available out of the box | + | `PluginAvailableTests.cs:20` | `heidiks/rabbitmq-delayed-message-exchange:3.13.0-management` | even a **non-bundled community plugin** (delayed message) is solved by a prebuilt image — no `.ez` download, no `rabbitmq-plugins enable`, no custom Dockerfile | + + So Bitween's fixture is simply the odd one out: `new RabbitMqBuilder().Build()` takes the + module's default image, which has **no** management plugin, so every §1.12 endpoint + (`/api/overview`, `/api/definitions`, `/api/queues`) would return nothing. Fix: + `.WithImage("rabbitmq:4-management")` and expose `15672`. Budget ~+50–100 MB and a slightly + slower readiness probe. + + **Switch the shared fixture rather than adding a second container** — the management API is also + the most convenient way to assert topology in *every* RabbitMQ test, not just the discovery ones. + + Two caveats worth a decision: + - `heidiks/…` is a **third-party image**. Fine for a test-only dependency, but pin it by digest + rather than tag, or replace it with a three-line in-repo Dockerfile + (`FROM rabbitmq:4-management` + `COPY` the `.ez` + `rabbitmq-plugins enable`) built through + Testcontainers' `ImageFromDockerfileBuilder`. For the *provider*, the delayed-plugin path only + matters if we choose to expose `SupportsRequeueDelay` (§1.7); the outbox/`DelayedRetry` path + doesn't need it. + - SW-Bus pins **3.13**; the official images are on 4.x. Align Bitween's fixture with whatever the + discovery code is developed against, since management-API response fields have grown across + 3.8 → 4.x (§1.12). +2. **The test process itself is a memory factor for Kafka.** `librdkafka` buffers are native and + per-partition (§2.3), so a test that subscribes to a multi-partition topic on defaults can + allocate hundreds of MB inside the test host. Set tiny buffers in test config + (`queued.max.messages.kbytes=1024`, `queued.min.messages=100`) — and add one test that asserts + the provider's preflight guard **rejects** an oversized configuration, so the guard rail from + §2.4 is covered rather than assumed. + +### 3.6 Local dev environment + +A `docker-compose.dev.yml` worth committing, because developing the discovery and Kafka features +without UIs is miserable: + +| Service | Why | +|---|---| +| `postgres` | app + Quartz store | +| `rabbitmq:4-management` (5672 + **15672**) | the management UI is both the dev tool *and* the thing §1.12 talks to | +| `redpanda` (tuned flags) | Kafka endpoint | +| `redpanda-console` | topics/groups/lag browser — the closest analogue to RabbitMQ's management UI, and effectively required to develop the Kafka provider | +| `eclipse-mosquitto` | ~15 MB, add it now for the MQTT phase | + +Two notes: the compose file should use the *same* image tags as the Testcontainers fixtures so dev +and CI don't diverge on broker version, and Redpanda Console is dev-only — never a dependency of +the test suite. + +### 3.7 Answering the "is it easy" question directly + +**Kafka's local/test story is easier than its runtime story.** Getting a broker up is a solved +problem: one tuned Redpanda container, ~5 seconds, ~200 MB, using the exact Testcontainers pattern +this repo already runs. The genuinely hard parts of Kafka are the ones in §2.2–2.4 — asynchronous +connection failures, rebalances interacting with reconcile, and per-partition native buffers — and +notably **none of those are made easier or harder by the choice of container**. So the test +environment should not influence the provider sequencing: pick Redpanda, spend the saved effort on +the rebalance and buffer-guard tests, which is where the real risk lives. From 55e8ebf7e21a6408aa075d082061981588a3fa3f Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Sun, 6 Sep 2026 04:42:06 +0300 Subject: [PATCH 02/43] test: integration tests for external bus providers, and read broker credentials from the container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends BitweenFixture with a SECOND RabbitMQ standing in for a customer's own broker — reusing the internal one would let a test pass while the message actually travelled Bitween's own bus, which is the confusion this feature exists to avoid — plus ElasticMQ for the SQS API without LocalStack's weight. Both bus adapters are published into the local cloud store with protocol-2 metadata, so the tests exercise install-from-storage rather than a local path. ExternalBusGatewayTests covers ingress becoming an Xchange, endpoint-to-gateway resolution, egress, staged test-connection, heartbeat health reaching the health view, and the regression that matters most: a gateway with no DataSourceId is still an internal-bus gateway. Fixed while running them: the fixture hardcoded guest/guest, but RabbitMqBuilder generates random credentials, so every external-broker test failed with ACCESS_REFUSED on PLAIN. Read from the container's own connection string now. STATUS: 4 of 8 passing. The transport works end to end — the adapter installs, attaches, connects, declares topology, consumes and heartbeats, which is what TestConnection and the health test prove. What does not yet work is downstream of that: no Xchange is created. Under investigation; the feature is not proven. Co-Authored-By: Claude Opus 5 --- .../Fixtures/AdapterInstaller.cs | 17 +- .../Fixtures/BitweenFixture.cs | 81 +++- .../Fixtures/BusAdapters.cs | 8 + .../SW.Bitween.IntegrationTests.csproj | 6 + .../Tests/ExternalBusGatewayTests.cs | 362 ++++++++++++++++++ 5 files changed, 467 insertions(+), 7 deletions(-) create mode 100644 SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/ExternalBusGatewayTests.cs diff --git a/SW.Bitween.IntegrationTests/Fixtures/AdapterInstaller.cs b/SW.Bitween.IntegrationTests/Fixtures/AdapterInstaller.cs index c322d88a..071420b7 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/AdapterInstaller.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/AdapterInstaller.cs @@ -15,7 +15,8 @@ internal static class AdapterInstaller Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "test-adapters"); public static async Task InstallAsync(ICloudFilesService cloudFiles, - string projectName, string adapterId, string entryAssembly) + string projectName, string adapterId, string entryAssembly, + IDictionary? extraMetadata = null) { var publishDir = Path.Combine(AdaptersRoot, projectName); @@ -37,16 +38,20 @@ public static async Task InstallAsync(ICloudFilesService cloudFiles, var bytes = zipStream.ToArray(); var hash = Convert.ToHexString(SHA256.HashData(bytes)).ToLower()[..16]; + var metadata = new Dictionary + { + { "EntryAssembly", entryAssembly }, + { "Hash", hash } + }; + foreach (var kv in extraMetadata ?? new Dictionary()) + metadata[kv.Key] = kv.Value; + using var uploadStream = new MemoryStream(bytes); await cloudFiles.WriteAsync(uploadStream, new WriteFileSettings { Key = $"adapters/{adapterId}".ToLower(), ContentType = "application/zip", - Metadata = new Dictionary - { - { "EntryAssembly", entryAssembly }, - { "Hash", hash } - } + Metadata = metadata }); } } diff --git a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs index 4943f8d3..ce0869e5 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs @@ -21,6 +21,8 @@ using SW.PrimitiveTypes; using SW.Scheduler; using SW.Serverless; +using SW.Serverless.Resident; +using SW.Bitween.Services.DataSources; using DotNet.Testcontainers.Builders; using DotNet.Testcontainers.Containers; using Testcontainers.PostgreSql; @@ -47,6 +49,21 @@ public sealed class BitweenFixture : IAsyncLifetime private readonly PostgreSqlContainer _postgres = new PostgreSqlBuilder().Build(); private readonly RabbitMqContainer _rabbitMq = new RabbitMqBuilder().Build(); + // A SECOND RabbitMQ, standing in for a customer's own broker. Reusing the internal one would + // let a test pass while the external path quietly published to Bitween's own bus, which is + // exactly the confusion the feature exists to avoid. + private readonly RabbitMqContainer _externalRabbitMq = new RabbitMqBuilder().Build(); + + // ElasticMQ speaks the SQS API without LocalStack's weight. The point is to exercise real + // receive/delete/visibility semantics rather than a mock that agrees with our assumptions. + private readonly IContainer _elasticMq = new ContainerBuilder() + .WithImage("softwaremill/elasticmq-native:1.6.11") + .WithPortBinding(ElasticMqContainerPort, true) + .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(ElasticMqContainerPort)) + .Build(); + + private const int ElasticMqContainerPort = 9324; + // A real SMTP server, because the one thing no unit test can prove about the alert feature is // that an actual handshake succeeds. Started here rather than expected on the developer's // machine: a test that quietly does nothing when a local service is missing reports a green run @@ -67,6 +84,42 @@ public sealed class BitweenFixture : IAsyncLifetime /// Base address of MailHog's own API, for reading back what was delivered. public string MailHogApi => $"http://{_mailHog.Hostname}:{_mailHog.GetMappedPublicPort(ApiContainerPort)}"; + /// + /// Connection details for the external broker, as a data source's properties. + /// + /// Read from the container's own connection string rather than assumed: RabbitMqBuilder + /// generates random credentials, so hardcoding guest/guest gets ACCESS_REFUSED. + /// + public Dictionary ExternalRabbitProperties => new() + { + ["Host"] = ExternalRabbitHost, + ["Port"] = ExternalRabbitPort.ToString(), + ["UserName"] = ExternalRabbitUser, + ["Password"] = ExternalRabbitPassword, + ["VirtualHost"] = "/", + ["DeclareMode"] = "create", + ["Prefetch"] = "8" + }; + + private Uri ExternalRabbitUri => new(_externalRabbitMq.GetConnectionString()); + + public string ExternalRabbitHost => ExternalRabbitUri.Host; + public int ExternalRabbitPort => ExternalRabbitUri.Port; + public string ExternalRabbitUser => ExternalRabbitUri.UserInfo.Split(':')[0]; + public string ExternalRabbitPassword => ExternalRabbitUri.UserInfo.Split(':') is [_, var p] ? p : ""; + + public string SqsServiceUrl => $"http://{_elasticMq.Hostname}:{_elasticMq.GetMappedPublicPort(ElasticMqContainerPort)}"; + + public Dictionary SqsProperties => new() + { + ["Region"] = "elasticmq", + ["ServiceUrl"] = SqsServiceUrl, + ["AccessKeyId"] = "x", + ["SecretAccessKey"] = "x", + ["WaitTimeSeconds"] = "1", + ["VisibilityTimeoutSeconds"] = "10" + }; + public IHost App { get; private set; } = null!; private ExceptionDispatchInfo? _initError; @@ -75,7 +128,8 @@ public async Task InitializeAsync() { try { - await Task.WhenAll(_postgres.StartAsync(), _rabbitMq.StartAsync(), _mailHog.StartAsync()); + await Task.WhenAll(_postgres.StartAsync(), _rabbitMq.StartAsync(), _mailHog.StartAsync(), + _externalRabbitMq.StartAsync(), _elasticMq.StartAsync()); var dataSourceBuilder = new NpgsqlDataSourceBuilder(_postgres.GetConnectionString()); dataSourceBuilder.EnableDynamicJson(); @@ -143,6 +197,18 @@ public async Task InitializeAsync() opts.AdapterLocalPath = Path.Combine(Path.GetTempPath(), "bitween-test-serverless"); }); + // The external bus provider runtime. The supervisor is deliberately NOT + // registered as a hosted service here: tests start data sources explicitly so + // they control timing, and BusProviderSupervisorTests drives it directly. + services.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/bitween-tests-{Environment.ProcessId}.sock"; + o.PipeName = $"bitween-tests-{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + o.MaxInFlight = 8; + }); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -184,6 +250,17 @@ await AdapterInstaller.InstallAsync(cloudFiles, "SW.Bitween.SampleHandler", "sw.bitween.samplehandler", "SW.Bitween.SampleHandler.dll"); await AdapterInstaller.InstallAsync(cloudFiles, "SW.Bitween.SampleConfigurableAdapter", "sw.bitween.sampleconfigurableadapter", "SW.Bitween.SampleConfigurableAdapter.dll"); + + // Protocol 2 metadata is what tells the host these are resident adapters rather + // than the classic per-invocation kind. + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.Adapters.Bus.RabbitMq", BusAdapters.RabbitMq, + "SW.Bitween.Adapters.Bus.RabbitMq.dll", + new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.Adapters.Bus.Sqs", BusAdapters.Sqs, + "SW.Bitween.Adapters.Bus.Sqs.dll", + new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); } await App.StartAsync(); @@ -211,6 +288,8 @@ public async Task DisposeAsync() await _postgres.DisposeAsync(); await _rabbitMq.DisposeAsync(); await _mailHog.DisposeAsync(); + await _externalRabbitMq.DisposeAsync(); + await _elasticMq.DisposeAsync(); } } diff --git a/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs b/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs new file mode 100644 index 00000000..19dbdbbb --- /dev/null +++ b/SW.Bitween.IntegrationTests/Fixtures/BusAdapters.cs @@ -0,0 +1,8 @@ +namespace SW.Bitween.IntegrationTests.Fixtures; + +/// Adapter ids the fixture installs, so tests and the fixture cannot drift apart. +public static class BusAdapters +{ + public const string RabbitMq = "bitween.bus.rabbitmq"; + public const string Sqs = "bitween.bus.sqs"; +} diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index 9f0e976d..6e569203 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -25,6 +25,10 @@ + + + - + diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index 6e569203..4ed90dc3 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -31,9 +31,7 @@ - - + diff --git a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj index 1ef6acb7..d360c7c1 100644 --- a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj +++ b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj @@ -5,6 +5,6 @@ SW.Bitween.SampleConfigurableAdapter - + diff --git a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj index 40c9124e..0e8b3ed8 100644 --- a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj +++ b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj index 2b87e8f9..4bff15fa 100644 --- a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj +++ b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj index aa6329d9..7af762f1 100644 --- a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj +++ b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj @@ -8,7 +8,7 @@ - + diff --git a/SW.Bitween.Web/SW.Bitween.Web.csproj b/SW.Bitween.Web/SW.Bitween.Web.csproj index e0131c98..220e9944 100644 --- a/SW.Bitween.Web/SW.Bitween.Web.csproj +++ b/SW.Bitween.Web/SW.Bitween.Web.csproj @@ -48,9 +48,7 @@ - - + diff --git a/SW.Bitween.sln b/SW.Bitween.sln index 21cd9456..9dd1d570 100644 --- a/SW.Bitween.sln +++ b/SW.Bitween.sln @@ -35,10 +35,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Bus Providers", "Bus Provid EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Bus.RabbitMq", "SW.Bitween.Adapters.Bus.RabbitMq\SW.Bitween.Adapters.Bus.RabbitMq.csproj", "{5E93D24F-EA1A-4788-B19F-326215B9CD2B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Sdk", "..\SW-Serverless\SW.Serverless.Sdk\SW.Serverless.Sdk.csproj", "{F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Contract", "..\SW-Serverless\SW.Serverless.Contract\SW.Serverless.Contract.csproj", "{A95E858E-25A6-4A5D-B10F-8D4E914A06F4}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Bus.Sqs", "SW.Bitween.Adapters.Bus.Sqs\SW.Bitween.Adapters.Bus.Sqs.csproj", "{1125FB09-88E3-405B-80C0-62C73B1AB9F8}" EndProject Global @@ -207,30 +203,6 @@ Global {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x64.Build.0 = Release|Any CPU {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x86.ActiveCfg = Release|Any CPU {5E93D24F-EA1A-4788-B19F-326215B9CD2B}.Release|x86.Build.0 = Release|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x64.ActiveCfg = Debug|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x64.Build.0 = Debug|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x86.ActiveCfg = Debug|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Debug|x86.Build.0 = Debug|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|Any CPU.Build.0 = Release|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x64.ActiveCfg = Release|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x64.Build.0 = Release|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x86.ActiveCfg = Release|Any CPU - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF}.Release|x86.Build.0 = Release|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x64.ActiveCfg = Debug|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x64.Build.0 = Debug|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x86.ActiveCfg = Debug|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Debug|x86.Build.0 = Debug|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|Any CPU.Build.0 = Release|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x64.ActiveCfg = Release|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x64.Build.0 = Release|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x86.ActiveCfg = Release|Any CPU - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4}.Release|x86.Build.0 = Release|Any CPU {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|Any CPU.Build.0 = Debug|Any CPU {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -256,8 +228,6 @@ Global {1474658D-E225-478E-80D6-D41A0376F88C} = {DCB20324-CBBC-43BB-9529-6F16C4033A5B} {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} = {5F58DD63-8ABF-4148-A594-0D9881F39142} {5E93D24F-EA1A-4788-B19F-326215B9CD2B} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} - {F5CE2A6F-30C8-464A-B11F-CB241CA5F6EF} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} - {A95E858E-25A6-4A5D-B10F-8D4E914A06F4} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} {1125FB09-88E3-405B-80C0-62C73B1AB9F8} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution From b3e4a7a40da6791f81081fa9afb81a0056abc7a3 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Sun, 6 Sep 2026 13:43:50 +0300 Subject: [PATCH 09/43] fix: five defects in the external bus gateway path, found by testing it properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconcile loop had no tests at all, despite a fixture comment claiming otherwise. Adding them, and pushing on the adapters under conditions that break a broker client rather than conditions that demonstrate one, turned up five real defects. 217 unit + 236 integration tests pass. ROUTING. BusProviderEventSink ordered candidate gateways by Endpoint descending to make an exact match beat the catch-all. PostgreSQL sorts NULLS FIRST on a descending order, so the catch-all won every time: a data source with both a catch-all and a specific gateway filed every message against the wrong Document, running the wrong subscriptions, with nothing in the audit trail saying so. Now ordered by the match itself, which means the same thing on all three providers. CONCURRENCY. RabbitBusHandler acked and nacked one shared IModel from concurrent thread-pool handlers, which RabbitMQ.Client does not support; only the publish channel was guarded, and even there CreateBasicProperties sat outside the lock. Both channels now have a gate, and shutdown goes through them too. IDENTITY. With no MessageId the adapter hashed the body for a dedupe key, so two legitimately identical messages — reorder the same SKU, the same daily totals — collapsed into one and the second was dropped for the whole deduplication window, thirty days by default. Unkeyed deliveries are handled once per delivery, with a warning, which trades a possible reprocess for never losing a real message. PROVIDERS. MySQL and SQL Server had no migrations for data_source, inbound_message or cluster_lease, so the feature only ever worked on PostgreSQL — MigrationDriftTests was already red on this branch, and its own remarks say the symptom is a pod crash-looping at startup. Generated both. CLUSTERING. ClusterLease was declared only in the PgSql context, so even with those migrations MySQL and SQL Server would have had no table for leader election, and two nodes could consume one queue silently. Moved to the base context and regenerated. Tests (21 new) - BusProviderSupervisorTests: start, endpoint derivation from gateways, a second gateway, no restart when nothing changed (the process id is the witness — a restart every thirty seconds would tear down every consumer twice a minute), restart when configuration changes, deactivate, lease release, placement, fencing, handover, one bad adapter not stopping the pass, health write-back, shutdown handover, inactive gateways. - RabbitBusAdapterTests: identical bodies, identity from the message id, integrity under concurrent delivery, concurrent publishes. - ExternalBusGatewayTests: exact match beats catch-all, catch-all still catches, and a data source feeding a gateway cannot be deleted. Five were mutation-checked: removing the fingerprint guard, the lease revalidation, the start-failure isolation and the inactive-gateway filter, and restoring the content-hash key, each make the intended test fail. Also: BusProviderSupervisor.ReconcileAsync is public — "reconcile now" is a real operation, the same shape as receivenow, and a loop whose only entry point is a thirty-second timer cannot be tested. BusGateway.EndpointProperties is passed through by the supervisor but read by no adapter; documented as inert rather than left to be discovered, and it no longer produces "Endpoint::key" for the catch-all. Corrected a stale comment in Startup.cs claiming placement across nodes was unimplemented. Co-Authored-By: Claude Opus 5 --- .../RabbitBusHandler.cs | 68 +- SW.Bitween.Api/Data/BitweenDbContext.cs | 11 + SW.Bitween.Api/Domain/Gateway/BusGateway.cs | 7 + .../DataSources/BusProviderEventSink.cs | 7 +- .../DataSources/BusProviderSupervisor.cs | 12 +- .../Tests/BusProviderSupervisorTests.cs | 658 +++++ .../Tests/ExternalBusGatewayTests.cs | 102 + .../Tests/RabbitBusAdapterTests.cs | 324 +++ ...6103810_ExternalBusDataSources.Designer.cs | 2290 +++++++++++++++++ .../20260906103810_ExternalBusDataSources.cs | 159 ++ .../BitweenDbContextModelSnapshot.cs | 152 ++ ...6103806_ExternalBusDataSources.Designer.cs | 2283 ++++++++++++++++ .../20260906103806_ExternalBusDataSources.cs | 178 ++ .../BitweenDbContextModelSnapshot.cs | 152 ++ SW.Bitween.Web/Startup.cs | 7 +- 15 files changed, 6386 insertions(+), 24 deletions(-) create mode 100644 SW.Bitween.IntegrationTests/Tests/BusProviderSupervisorTests.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/RabbitBusAdapterTests.cs create mode 100644 SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.Designer.cs create mode 100644 SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.cs create mode 100644 SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.Designer.cs create mode 100644 SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.cs diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs index a33809a7..1a2329f5 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; @@ -33,6 +32,12 @@ public class RabbitBusHandler : IResidentAdapter private IConnection _connection; private IModel _consumeChannel; private IModel _publishChannel; + + // RabbitMQ.Client does not support concurrent application operations on one IModel, and every + // delivery is handled on the thread pool — so acks, nacks and the shutdown calls all go through + // these. Two gates rather than one: a publish must never queue behind a slow ack. + private readonly object _consumeGate = new(); + private readonly object _publishGate = new(); private CancellationTokenSource _stopping; private readonly List _endpoints = new(); @@ -139,10 +144,16 @@ public Task StopAsync(CancellationToken cancellationToken) _state = "Draining"; _stopping?.Cancel(); - foreach (var tag in _consumerTags.Values) - try { _consumeChannel?.BasicCancel(tag); } catch { } + lock (_consumeGate) + { + foreach (var tag in _consumerTags.Values) + try { _consumeChannel?.BasicCancel(tag); } catch { } + + try { _consumeChannel?.Close(); } catch { } + } - try { _consumeChannel?.Close(); _publishChannel?.Close(); } catch { } + lock (_publishGate) + try { _publishChannel?.Close(); } catch { } try { _connection?.Close(TimeSpan.FromSeconds(3)); } catch { } _connection?.Dispose(); @@ -206,11 +217,15 @@ private async Task HandleAsync(string endpoint, BasicDeliverEventArgs delivery) var result = await _context.PublishAsync( delivery.Body, - // The broker's own message id when it has one, otherwise a content hash. NOT the - // delivery tag: tags are per channel and restart at 1 on every reconnect. + // The broker's own message id, and nothing else. NOT the delivery tag — tags are + // per channel and restart at 1 on every reconnect — and NOT a content hash: two + // messages that legitimately carry the same body are two messages, and hashing + // them would silently drop the second for the whole deduplication window. An + // unkeyed delivery is handled once per delivery, which is the honest answer when + // the publisher gave us nothing to identify it by. dedupeKey: delivery.BasicProperties?.MessageId is { Length: > 0 } id ? $"rabbit:{_options.Host}:{endpoint}:{id}" - : $"rabbit:{_options.Host}:{endpoint}:{Convert.ToHexString(SHA256.HashData(delivery.Body.Span))[..32]}", + : WarnUnkeyed(endpoint), endpoint: endpoint, headers: headers, contentType: delivery.BasicProperties?.ContentType ?? "application/json", @@ -218,14 +233,14 @@ private async Task HandleAsync(string endpoint, BasicDeliverEventArgs delivery) if (result.Accepted) { - _consumeChannel.BasicAck(delivery.DeliveryTag, multiple: false); + lock (_consumeGate) _consumeChannel.BasicAck(delivery.DeliveryTag, multiple: false); Interlocked.Increment(ref _acked); _lastMessageOn = DateTimeOffset.UtcNow; _context.Metric("bitween.bus.rabbitmq.acked", 1); } else { - _consumeChannel.BasicNack(delivery.DeliveryTag, multiple: false, requeue: true); + lock (_consumeGate) _consumeChannel.BasicNack(delivery.DeliveryTag, multiple: false, requeue: true); Interlocked.Increment(ref _nacked); _lastError = result.Error; _logger.LogWarning("Bitween rejected a message from {Endpoint}: {Error}. Requeued.", @@ -234,17 +249,29 @@ private async Task HandleAsync(string endpoint, BasicDeliverEventArgs delivery) } catch (OperationCanceledException) { - try { _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } + try { lock (_consumeGate) _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } } catch (Exception ex) { Interlocked.Increment(ref _failed); _lastError = ex.Message; _logger.LogError(ex, "Failed to hand a delivery from {Endpoint} to Bitween.", endpoint); - try { _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } + try { lock (_consumeGate) _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } } } + private long _unkeyed; + + private string WarnUnkeyed(string endpoint) + { + if (Interlocked.Increment(ref _unkeyed) == 1) + _logger.LogWarning( + "A message arrived on {Endpoint} with no MessageId, so it cannot be deduplicated. " + + "A redelivery of it will be processed again. Publishers should set one.", endpoint); + + return null; + } + // ---------------------------------------------------------------- commands /// @@ -256,23 +283,28 @@ public Task Publish(PublishRequest request) if (string.IsNullOrWhiteSpace(request?.Endpoint) && string.IsNullOrWhiteSpace(request?.Exchange)) throw new ArgumentException("Either Endpoint or Exchange is required."); - var properties = _publishChannel.CreateBasicProperties(); - properties.ContentType = request.ContentType ?? "application/json"; - properties.MessageId = request.MessageId ?? Guid.NewGuid().ToString("N"); - properties.DeliveryMode = (byte)(_options.Durable ? 2 : 1); - var body = System.Text.Encoding.UTF8.GetBytes(request.Body ?? ""); + var messageId = request.MessageId ?? Guid.NewGuid().ToString("N"); + + // CreateBasicProperties is itself a channel operation, so it belongs inside the gate with + // the publish rather than beside it. + lock (_publishGate) + { + var properties = _publishChannel.CreateBasicProperties(); + properties.ContentType = request.ContentType ?? "application/json"; + properties.MessageId = messageId; + properties.DeliveryMode = (byte)(_options.Durable ? 2 : 1); - lock (_publishChannel) _publishChannel.BasicPublish( exchange: request.Exchange ?? "", routingKey: request.Exchange == null ? request.Endpoint : request.RoutingKey ?? "", mandatory: false, basicProperties: properties, body: body); + } Interlocked.Increment(ref _published); - return Task.FromResult(new { messageId = properties.MessageId, bytes = body.Length }); + return Task.FromResult(new { messageId, bytes = body.Length }); } /// The control the UI needs before a data source is saved. Staged, so a failure names the step. diff --git a/SW.Bitween.Api/Data/BitweenDbContext.cs b/SW.Bitween.Api/Data/BitweenDbContext.cs index 209204ae..74d2990f 100644 --- a/SW.Bitween.Api/Data/BitweenDbContext.cs +++ b/SW.Bitween.Api/Data/BitweenDbContext.cs @@ -155,6 +155,17 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ds.HasIndex(p => p.Name).IsUnique(); }); + // Declared here rather than only in the PgSql context: leader election needs this table + // on every provider Bitween supports, and a node whose database has no cluster_lease + // cannot fence anything — which means two nodes can consume one queue, silently. + modelBuilder.Entity(cl => + { + cl.ToTable("ClusterLeases"); + cl.HasKey(i => i.Id); + cl.Property(i => i.Id).HasMaxLength(200).IsUnicode(false); + cl.Property(i => i.OwnerNode).HasMaxLength(200).IsUnicode(false); + }); + modelBuilder.Entity(im => { im.ToTable("InboundMessages"); diff --git a/SW.Bitween.Api/Domain/Gateway/BusGateway.cs b/SW.Bitween.Api/Domain/Gateway/BusGateway.cs index 22452677..31909816 100644 --- a/SW.Bitween.Api/Domain/Gateway/BusGateway.cs +++ b/SW.Bitween.Api/Domain/Gateway/BusGateway.cs @@ -33,6 +33,13 @@ public class BusGateway : BaseEntity, IAudited /// Per-gateway overrides handed to the adapter alongside the data source's own properties — /// prefetch, consumer group, visibility timeout. Connection settings belong on the DataSource; /// these are about this one subscription to it. + /// + /// NOT YET CONSUMED. The supervisor namespaces these and passes them through as + /// Endpoint:<endpoint>:<key> startup values, but neither bundled adapter reads + /// them, so setting one currently has no effect. The transport is in place; binding them is + /// per-adapter work — RabbitMQ can apply queue type and durability per endpoint at declare + /// time, SQS a per-queue visibility timeout, and prefetch would need a consume channel per + /// endpoint rather than the single one it has now. /// public Dictionary EndpointProperties { get; set; } = new(); diff --git a/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs b/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs index 45c7a7d3..d73a9e05 100644 --- a/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs +++ b/SW.Bitween.Api/Services/DataSources/BusProviderEventSink.cs @@ -55,7 +55,12 @@ public async Task OnEventAsync(InboundEvent inboundEvent, Cancella var gateway = await dbContext.Set() .Where(g => g.DataSourceId == dataSourceId && !g.Inactive) .Where(g => g.Endpoint == inboundEvent.Endpoint || g.Endpoint == null) - .OrderByDescending(g => g.Endpoint) // an exact endpoint match beats the catch-all + // An exact endpoint match beats the catch-all. Ordering by the endpoint itself does + // NOT express that: PostgreSQL sorts NULLS FIRST on a descending order, so the + // catch-all would win every race and every specific endpoint's messages would be + // filed against the wrong Document. Order by the match itself instead — true first, + // and it means the same thing on all three providers. + .OrderByDescending(g => g.Endpoint == inboundEvent.Endpoint) .FirstOrDefaultAsync(cancellationToken); if (gateway == null) diff --git a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs index a8100cf8..1d7a2813 100644 --- a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs +++ b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs @@ -91,7 +91,12 @@ public override async Task StopAsync(CancellationToken cancellationToken) await ReleaseAsync(dataSourceId); } - private async Task ReconcileAsync(CancellationToken cancellationToken) + /// + /// One reconciliation pass. Public because "reconcile now" is a real operation — the same + /// shape as the receivenow endpoint for subscriptions — and because a supervisor whose only + /// entry point is a thirty-second timer cannot be tested at all. + /// + public async Task ReconcileAsync(CancellationToken cancellationToken = default) { using var scope = _serviceProvider.CreateScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); @@ -250,7 +255,10 @@ private static Dictionary BuildStartupValues(DataSource dataSour if (wanted.Count > 0) values["Endpoints"] = string.Join(",", wanted); // Per-gateway overrides, namespaced so they cannot collide with connection settings. - foreach (var gateway in mine) + // See BusGateway.EndpointProperties: no bundled adapter reads these yet. Skipping the + // catch-all gateway is deliberate — an override with no endpoint to attach to would land + // under the meaningless key "Endpoint::something". + foreach (var gateway in mine.Where(g => !string.IsNullOrWhiteSpace(g.Endpoint))) foreach (var kv in gateway.EndpointProperties ?? new()) values[$"Endpoint:{gateway.Endpoint}:{kv.Key}"] = kv.Value; diff --git a/SW.Bitween.IntegrationTests/Tests/BusProviderSupervisorTests.cs b/SW.Bitween.IntegrationTests/Tests/BusProviderSupervisorTests.cs new file mode 100644 index 00000000..80b73aad --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/BusProviderSupervisorTests.cs @@ -0,0 +1,658 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using SW.Bitween.Domain; +using SW.Bitween.Domain.Cluster; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.Bitween.Services.Cluster; +using SW.Bitween.Services.DataSources; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The supervisor: the loop that decides WHICH broker connections this node holds open. +/// +/// Everything else in the external-bus feature has been exercised through an adapter started by +/// hand. That skips the part most likely to go wrong in production, because reconciliation is not a +/// one-shot: it runs every thirty seconds, for ever, against a database other people are editing. +/// The failures it can produce are the expensive kind — an adapter restarted on every pass drops +/// in-flight messages, a lease held after it was superseded means two nodes consuming one queue, +/// and one unreachable broker taking the loop down with it stops every other integration on the +/// node. +/// +/// So these tests drive ReconcileAsync directly and ask what the loop actually did, rather than +/// waiting on a timer and hoping. +/// +[Collection("Bitween")] +public class BusProviderSupervisorTests +{ + private readonly BitweenFixture _fixture; + + public BusProviderSupervisorTests(BitweenFixture fixture) => _fixture = fixture; + + /// + /// The whole point, end to end: a row in the database becomes a live broker connection, and a + /// message on that broker becomes an Xchange — with nobody starting an adapter by hand. + /// + [Fact] + public async Task Reconciling_starts_the_adapter_for_an_active_data_source() + { + var queue = Unique("sup-start"); + var dataSourceId = await CreateDataSourceAsync(); + var documentId = await AddGatewayAsync(dataSourceId, queue); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + Assert.NotNull(InstanceOf(dataSourceId)); + + Publish(queue, """{"from":"the supervisor"}"""); + + var xchange = await WaitForXchangeAsync(documentId); + Assert.NotNull(xchange); + } + + /// + /// The endpoint comes from the GATEWAY, not from the data source. This is what lets one + /// connection serve many queues, and it is the only reason a data source needs no queue + /// configuration of its own. + /// + [Fact] + public async Task A_gateway_endpoint_is_what_tells_the_adapter_which_queue_to_consume() + { + var queue = Unique("sup-endpoint"); + var dataSourceId = await CreateDataSourceAsync(); + var documentId = await AddGatewayAsync(dataSourceId, queue); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var declared = await DetailsOf(dataSourceId); + Assert.Contains(queue, string.Join(",", declared.Values)); + + Publish(queue, """{"routed":"by endpoint"}"""); + Assert.NotNull(await WaitForXchangeAsync(documentId)); + } + + /// + /// A second gateway on the same connection must reach the adapter, and the first must keep + /// working. Restarting for a new queue is acceptable; losing the old one is not. + /// + [Fact] + public async Task A_second_gateway_adds_its_queue_without_losing_the_first() + { + var first = Unique("sup-two-a"); + var second = Unique("sup-two-b"); + + var dataSourceId = await CreateDataSourceAsync(); + var firstDocument = await AddGatewayAsync(dataSourceId, first); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var secondDocument = await AddGatewayAsync(dataSourceId, second); + await supervisor.ReconcileAsync(); + + Publish(first, """{"queue":"first"}"""); + Publish(second, """{"queue":"second"}"""); + + Assert.NotNull(await WaitForXchangeAsync(firstDocument)); + Assert.NotNull(await WaitForXchangeAsync(secondDocument)); + } + + /// + /// The most consequential thing this loop does is NOTHING. Reconciliation runs every thirty + /// seconds; if an unchanged data source were restarted each pass, every long-running consumer + /// on the node would be torn down twice a minute and whatever it held in flight would be + /// redelivered. The process id is the honest witness — a restart cannot preserve it. + /// + [Fact] + public async Task An_unchanged_data_source_is_left_alone_across_passes() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-stable")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var before = InstanceOf(dataSourceId)?.ProcessId; + Assert.NotNull(before); + + await supervisor.ReconcileAsync(); + await supervisor.ReconcileAsync(); + + Assert.Equal(before, InstanceOf(dataSourceId)?.ProcessId); + } + + /// + /// A changed connection, on the other hand, MUST restart: an adapter holding a connection built + /// from the old credentials is exactly the stale state the loop exists to correct. + /// + [Fact] + public async Task A_changed_data_source_restarts_its_adapter() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-change")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var before = InstanceOf(dataSourceId)?.ProcessId; + Assert.NotNull(before); + + await MutateAsync(dataSourceId, d => d.Properties["Prefetch"] = "3"); + await supervisor.ReconcileAsync(); + + var after = InstanceOf(dataSourceId)?.ProcessId; + Assert.NotNull(after); + Assert.NotEqual(before, after); + } + + /// Deactivating is how an operator stops a connection without losing its configuration. + [Fact] + public async Task Deactivating_a_data_source_stops_its_adapter() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-inactive")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + Assert.NotNull(InstanceOf(dataSourceId)); + + await MutateAsync(dataSourceId, d => d.Inactive = true); + await supervisor.ReconcileAsync(); + + Assert.Null(InstanceOf(dataSourceId)); + } + + /// + /// Stopping is not enough — the lease has to go too. A node holding a lock on a data source + /// nobody is running would block whichever node later wants it, and nothing would ever + /// release it. + /// + [Fact] + public async Task Deactivating_a_data_source_releases_its_lease() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-release")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + await MutateAsync(dataSourceId, d => d.Inactive = true); + await supervisor.ReconcileAsync(); + + // Another node must now be able to take it. Retried briefly: the broker releases an + // exclusive queue promptly, but not instantly. + using var other = Node(); + var lease = await WaitForAcquireAsync(other, $"datasource.{dataSourceId}"); + + Assert.NotNull(lease); + await lease!.DisposeAsync(); + } + + /// + /// Placement. A data source another node already owns must not be started here, or both nodes + /// consume the same queue and every message is processed twice — the failure this whole design + /// exists to prevent. + /// + [Fact] + public async Task A_data_source_owned_by_another_node_is_not_started_here() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-owned")); + + using var otherNode = Node(); + await using var theirs = await otherNode.TryAcquireAsync($"datasource.{dataSourceId}"); + Assert.NotNull(theirs); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + Assert.Null(InstanceOf(dataSourceId)); + } + + /// + /// Fencing, which is the case a lock alone cannot handle. A node paused long enough for its + /// lease to be superseded still believes it holds ownership; the database term is what tells it + /// otherwise. Bumping the term stands in for that pause. + /// + /// What the supervisor must do is STOP — immediately, and without draining, because another + /// node may already be consuming and finishing in-flight work here would process the same + /// messages twice. The process id is the witness: it can only change if the adapter was torn + /// down and started again. + /// + /// It then starts again under a new term, and that is correct rather than a miss. Losing the + /// term does not mean the resource is taken — it means this node's claim is no longer proof + /// that it isn't. So it drops everything and asks again. Here nobody else holds the broker + /// lock, so it legitimately wins; when another node really does hold it, the re-acquire fails + /// and the adapter stays down — which is what + /// covers. + /// + [Fact] + public async Task A_superseded_lease_stops_the_adapter_before_anything_else() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-fence")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var before = InstanceOf(dataSourceId)?.ProcessId; + Assert.NotNull(before); + var termBefore = await TermOf($"datasource.{dataSourceId}"); + + // Someone else won the resource while this node was not looking. + await using (var scope = _fixture.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var row = await db.Set().FirstAsync(l => l.Id == $"datasource.{dataSourceId}"); + row.Claim("some-other-node"); + await db.SaveChangesAsync(); + } + + await supervisor.ReconcileAsync(); + + var after = InstanceOf(dataSourceId)?.ProcessId; + Assert.NotNull(after); + Assert.NotEqual(before, after); + + // And it did not simply carry on under the stale claim. + Assert.True(await TermOf($"datasource.{dataSourceId}") > termBefore + 1, + "the supervisor kept its superseded term instead of taking a new one"); + } + + /// + /// The handover this node is on the losing side of. Once another node holds the resource, an + /// adapter still running here is duplicate processing — so it has to go, even though the data + /// source is perfectly active and perfectly healthy. + /// + /// StopAsync releases the leases but leaves the supervisor's idea of what it is running intact, + /// which is exactly the state a node is in after a pause: still consuming, no longer entitled + /// to. + /// + [Fact] + public async Task A_data_source_taken_over_by_another_node_is_stopped_here() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-taken")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + Assert.NotNull(InstanceOf(dataSourceId)); + + // Give up the lock without stopping the adapter, then let another node take it. + await supervisor.StopAsync(default); + + using var otherNode = Node(); + await using var theirs = await WaitForAcquireAsync(otherNode, $"datasource.{dataSourceId}"); + Assert.NotNull(theirs); + + await supervisor.ReconcileAsync(); + + Assert.Null(InstanceOf(dataSourceId)); + } + + /// + /// One broker being unreachable is a normal Tuesday. If it could stop the pass, a single + /// customer's expired credentials would take every other integration on the node down with it. + /// + [Fact] + public async Task One_data_source_that_cannot_start_does_not_stop_the_others() + { + var broken = await CreateDataSourceAsync(adapterId: "bitween.bus.doesnotexist"); + await AddGatewayAsync(broken, Unique("sup-broken")); + + var queue = Unique("sup-healthy"); + var healthy = await CreateDataSourceAsync(); + var documentId = await AddGatewayAsync(healthy, queue); + + await using var supervisor = Supervisor(); + + // Not "does not throw" — the pass has to complete far enough to start the good one. + await supervisor.ReconcileAsync(); + + Assert.Null(InstanceOf(broken)); + Assert.NotNull(InstanceOf(healthy)); + + Publish(queue, """{"still":"working"}"""); + Assert.NotNull(await WaitForXchangeAsync(documentId)); + } + + /// + /// Health has to land in the database, because that is the only place the UI and the notifiers + /// can see it. An operator should not need to tail logs to find out a broker went away. + /// + [Fact] + public async Task Health_and_ownership_are_written_back_to_the_data_source() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-health")); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + // The first pass starts the adapter; health arrives on a heartbeat, so it is the pass + // after the first heartbeat that records it. + DataSource row = null; + await WaitAsync(async () => + { + await supervisor.ReconcileAsync(); + row = await ReadAsync(dataSourceId); + return row.LastHeartbeatOn != null; + }, TimeSpan.FromSeconds(30), "the heartbeat never reached the data source row"); + + Assert.NotNull(row!.LastKnownState); + Assert.NotNull(row.OwnedByNode); + Assert.Contains("term", row.OwnedByNode); + } + + /// + /// Shutdown is a handover, not an abandonment. Releasing on the way out is what makes a rolling + /// restart take seconds instead of waiting for the broker to time the old connection out. + /// + [Fact] + public async Task Stopping_the_supervisor_releases_what_it_held() + { + var dataSourceId = await CreateDataSourceAsync(); + await AddGatewayAsync(dataSourceId, Unique("sup-handover")); + + var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + Assert.NotNull(InstanceOf(dataSourceId)); + + await supervisor.StopAsync(default); + supervisor.Dispose(); + + using var other = Node(); + var lease = await WaitForAcquireAsync(other, $"datasource.{dataSourceId}"); + Assert.NotNull(lease); + await lease!.DisposeAsync(); + + await StopInstanceAsync(dataSourceId); + } + + /// + /// An inactive gateway must not contribute its endpoint. Otherwise deactivating a gateway would + /// leave Bitween still consuming its queue and acking messages nobody routes anywhere — a + /// silent drop, which is worse than an error. + /// + [Fact] + public async Task An_inactive_gateway_does_not_contribute_its_endpoint() + { + var live = Unique("sup-live"); + var dead = Unique("sup-dead"); + + var dataSourceId = await CreateDataSourceAsync(); + var liveDocument = await AddGatewayAsync(dataSourceId, live); + var deadGatewayId = await AddGatewayAsync(dataSourceId, dead, returnGatewayId: true); + + await using (var scope = _fixture.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var gateway = await db.Set().FirstAsync(g => g.Id == deadGatewayId); + gateway.Inactive = true; + await db.SaveChangesAsync(); + scope.ServiceProvider.GetRequiredService().Revoke(); + } + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var details = string.Join(",", (await DetailsOf(dataSourceId)).Values); + Assert.Contains(live, details); + Assert.DoesNotContain(dead, details); + + // And prove it by behaviour: the live queue still works. + Publish(live, """{"gateway":"live"}"""); + Assert.NotNull(await WaitForXchangeAsync(liveDocument)); + } + + // ---------------------------------------------------------------- helpers + + private static string Unique(string prefix) => $"{prefix}-{Guid.NewGuid():N}"[..24]; + + /// + /// A supervisor with its own election instance, so it stands in for one node. Disposing stops + /// whatever it started, which keeps a failing test from leaving a live broker connection behind + /// for the rest of the collection. + /// + private TestSupervisor Supervisor() => new( + new BusProviderSupervisor( + _fixture.App.Services, + _fixture.App.Services.GetRequiredService(), + Node(), + _fixture.App.Services.GetRequiredService() + .CreateLogger()), + _fixture.App.Services.GetRequiredService()); + + private sealed class TestSupervisor : IAsyncDisposable + { + private readonly BusProviderSupervisor _supervisor; + private readonly IResidentAdapterHost _adapters; + + // Every test in the collection shares one resident host, so cleaning up "whatever is + // running" would stop another test's adapter. Only what appeared after this supervisor + // was built is ours to stop. + private readonly HashSet _preexisting; + private readonly HashSet _started = new(); + + public TestSupervisor(BusProviderSupervisor supervisor, IResidentAdapterHost adapters) + { + _supervisor = supervisor; + _adapters = adapters; + _preexisting = Keys(); + } + + private HashSet Keys() => + _adapters.Describe().Select(h => $"{h.AdapterId}|{h.InstanceKey}").ToHashSet(); + + public async Task ReconcileAsync() + { + await _supervisor.ReconcileAsync(); + foreach (var key in Keys().Where(k => !_preexisting.Contains(k))) + _started.Add(key); + } + + public Task StopAsync(System.Threading.CancellationToken token) => _supervisor.StopAsync(token); + + public void Dispose() => _supervisor.Dispose(); + + public async ValueTask DisposeAsync() + { + try { await _supervisor.StopAsync(default); } catch { /* best effort */ } + + var live = Keys(); + foreach (var key in _started.Where(live.Contains)) + { + var parts = key.Split('|'); + try { await _adapters.StopAsync(parts[0], parts[1], drain: false); } catch { } + } + + _supervisor.Dispose(); + } + } + + private RabbitMqLeaderElection Node() => new( + _fixture.App.Services.GetRequiredService(), + _fixture.App.Services, + _fixture.App.Services.GetRequiredService() + .CreateLogger()); + + private static async Task WaitForAcquireAsync(ILeaderElection node, string resource) + { + var deadline = DateTime.UtcNow.AddSeconds(20); + while (DateTime.UtcNow < deadline) + { + var lease = await node.TryAcquireAsync(resource); + if (lease != null) return lease; + await Task.Delay(250); + } + return null; + } + + /// + /// A data source with connection settings ONLY. No Endpoints key: the whole point of these + /// tests is that the supervisor derives that from the gateways. + /// + private async Task CreateDataSourceAsync(string adapterId = null) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var dataSource = new DataSource + { + Name = Unique("ds"), + AdapterId = adapterId ?? BusAdapters.RabbitMq, + Kind = DataSourceKind.Broker, + Properties = new Dictionary(_fixture.ExternalRabbitProperties) + }; + + db.Add(dataSource); + await db.SaveChangesAsync(); + return dataSource.Id; + } + + private async Task AddGatewayAsync(int dataSourceId, string endpoint, bool returnGatewayId = false) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var document = new Document(null, Unique("doc"), DocumentFormat.Json); + db.Add(document); + await db.SaveChangesAsync(); + + var gateway = new BusGateway + { + Name = Unique("gw"), + DocumentId = document.Id, + DataSourceId = dataSourceId, + Endpoint = endpoint + }; + db.Add(gateway); + await db.SaveChangesAsync(); + + // The infolink cache is a singleton holding a ten-minute snapshot; without revoking it a + // brand-new Document resolves to null and the message is acked and silently dropped. + scope.ServiceProvider.GetRequiredService().Revoke(); + + return returnGatewayId ? gateway.Id : document.Id; + } + + private async Task MutateAsync(int dataSourceId, Action mutate) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var row = await db.Set().FirstAsync(d => d.Id == dataSourceId); + mutate(row); + await db.SaveChangesAsync(); + } + + private async Task TermOf(string resource) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var row = await db.Set().AsNoTracking().FirstOrDefaultAsync(l => l.Id == resource); + return row?.Term ?? 0; + } + + private async Task ReadAsync(int dataSourceId) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + return await db.Set().AsNoTracking().FirstAsync(d => d.Id == dataSourceId); + } + + private InstanceHealth InstanceOf(int dataSourceId) => + _fixture.App.Services.GetRequiredService() + .Describe().FirstOrDefault(h => h.InstanceKey == dataSourceId.ToString()); + + /// + /// What the adapter says it is consuming. Read from the heartbeat rather than from the startup + /// values, so it reflects what actually reached the broker. + /// + private async Task> DetailsOf(int dataSourceId) + { + IDictionary details = new Dictionary(); + await WaitAsync(() => + { + var instance = InstanceOf(dataSourceId); + if (instance?.Details is not { Count: > 0 }) return Task.FromResult(false); + details = instance.Details; + return Task.FromResult(true); + }, TimeSpan.FromSeconds(30), $"no heartbeat detail arrived for data source {dataSourceId}"); + + return details; + } + + private async Task StopInstanceAsync(int dataSourceId) + { + var instance = InstanceOf(dataSourceId); + if (instance == null) return; + + var host = _fixture.App.Services.GetRequiredService(); + try { await host.StopAsync(instance.AdapterId, instance.InstanceKey, drain: false); } catch { } + } + + private void Publish(string queue, string body) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + + channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false); + + var properties = channel.CreateBasicProperties(); + properties.ContentType = "application/json"; + properties.MessageId = Guid.NewGuid().ToString("N"); + properties.DeliveryMode = 2; + + channel.BasicPublish("", queue, properties, Encoding.UTF8.GetBytes(body)); + } + + private IConnection ExternalConnection() => new ConnectionFactory + { + HostName = _fixture.ExternalRabbitHost, + Port = _fixture.ExternalRabbitPort, + UserName = _fixture.ExternalRabbitUser, + Password = _fixture.ExternalRabbitPassword + }.CreateConnection("supervisor-tests"); + + private async Task WaitForXchangeAsync(int documentId) + { + Xchange found = null; + await WaitAsync(async () => + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + found = await db.Set().AsNoTracking() + .FirstOrDefaultAsync(x => x.DocumentId == documentId); + return found != null; + }, TimeSpan.FromSeconds(45), $"no Xchange was created for document {documentId}"); + + return found; + } + + private static async Task WaitAsync(Func> condition, TimeSpan timeout, string because) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + try { if (await condition()) return; } catch { /* still settling */ } + await Task.Delay(300); + } + Assert.Fail($"Timed out after {timeout}: {because}"); + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/ExternalBusGatewayTests.cs b/SW.Bitween.IntegrationTests/Tests/ExternalBusGatewayTests.cs index 23228b2b..fd9641a4 100644 --- a/SW.Bitween.IntegrationTests/Tests/ExternalBusGatewayTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/ExternalBusGatewayTests.cs @@ -154,6 +154,108 @@ public async Task Each_gateway_receives_only_its_own_endpoint() Assert.Equal(0, await db.Set().CountAsync(x => x.DocumentId == invoiceDoc)); } + /// + /// A gateway with no endpoint is the catch-all: it claims whatever no other gateway does. The + /// exact match has to win, or a data source that gains a catch-all silently starts routing + /// every specific endpoint's messages to the wrong Document — running the wrong subscriptions, + /// against the wrong mapping, with nothing in the audit trail saying so. + /// + /// Driven through the sink rather than the broker: this is a routing decision, and the queue + /// only adds latency to it. + /// + [Fact] + public async Task An_exact_endpoint_match_beats_the_catch_all_gateway() + { + var invoices = Unique("catchall-x"); + + var dataSourceId = await CreateDataSourceAsync(invoices, withGateway: false); + var catchAllDoc = await AddGatewayAsync(dataSourceId, endpoint: null); + var invoiceDoc = await AddGatewayAsync(dataSourceId, invoices); + + var sink = _fixture.App.Services.GetRequiredService(); + + var outcome = await sink.OnEventAsync(new InboundEvent + { + AdapterId = BusAdapters.RabbitMq, + InstanceKey = dataSourceId.ToString(), + Endpoint = invoices, + DedupeKey = Guid.NewGuid().ToString("N"), + Payload = Encoding.UTF8.GetBytes("{\"invoiceId\":9}") + }, CancellationToken.None); + + Assert.True(outcome.Accepted, outcome.Error); + + await using var scope = _fixture.App.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + Assert.Equal(1, await db.Set().CountAsync(x => x.DocumentId == invoiceDoc)); + Assert.Equal(0, await db.Set().CountAsync(x => x.DocumentId == catchAllDoc)); + } + + /// + /// And the catch-all still catches what nothing else claims — otherwise the fix above would + /// just be a way of disabling it. + /// + [Fact] + public async Task The_catch_all_gateway_still_claims_an_unmatched_endpoint() + { + var known = Unique("catchall-k"); + + var dataSourceId = await CreateDataSourceAsync(known, withGateway: false); + var catchAllDoc = await AddGatewayAsync(dataSourceId, endpoint: null); + var knownDoc = await AddGatewayAsync(dataSourceId, known); + + var sink = _fixture.App.Services.GetRequiredService(); + + var outcome = await sink.OnEventAsync(new InboundEvent + { + AdapterId = BusAdapters.RabbitMq, + InstanceKey = dataSourceId.ToString(), + Endpoint = Unique("unclaimed"), + DedupeKey = Guid.NewGuid().ToString("N"), + Payload = Encoding.UTF8.GetBytes("{\"stray\":true}") + }, CancellationToken.None); + + Assert.True(outcome.Accepted, outcome.Error); + + await using var scope = _fixture.App.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + Assert.Equal(1, await db.Set().CountAsync(x => x.DocumentId == catchAllDoc)); + Assert.Equal(0, await db.Set().CountAsync(x => x.DocumentId == knownDoc)); + } + + /// + /// Deleting a data source that still feeds a gateway must be refused. + /// + /// The alternative is worse than an error: a nullable foreign key that quietly becomes null + /// turns that gateway back into an INTERNAL bus gateway, and Bitween starts consuming its own + /// bus for a Document that was configured to read a customer's broker. Nothing fails, nothing + /// logs, and the integration is simply pointed somewhere else. + /// + [Fact] + public async Task A_data_source_still_feeding_a_gateway_cannot_be_deleted() + { + var queue = Unique("delete-guard"); + var dataSourceId = await CreateDataSourceAsync(queue, withGateway: false); + await AddGatewayAsync(dataSourceId, queue); + + await using var scope = _fixture.App.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + db.Remove(await db.Set().FirstAsync(d => d.Id == dataSourceId)); + await Assert.ThrowsAnyAsync(() => db.SaveChangesAsync()); + + // And the gateway is untouched — still external, still on its data source. + await using var check = _fixture.App.Services.CreateAsyncScope(); + var fresh = check.ServiceProvider.GetRequiredService(); + var gateway = await fresh.Set().AsNoTracking() + .FirstAsync(g => g.DataSourceId == dataSourceId); + + Assert.Equal(dataSourceId, gateway.DataSourceId); + Assert.Equal(queue, gateway.Endpoint); + } + // ---------------------------------------------------------------- egress [Fact] diff --git a/SW.Bitween.IntegrationTests/Tests/RabbitBusAdapterTests.cs b/SW.Bitween.IntegrationTests/Tests/RabbitBusAdapterTests.cs new file mode 100644 index 00000000..6eec222b --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/RabbitBusAdapterTests.cs @@ -0,0 +1,324 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using RabbitMQ.Client; +using SW.Bitween.Domain; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The ingress adapter itself, under the conditions that break a broker client rather than the +/// conditions that demonstrate one. +/// +/// The gateway tests prove a message arrives. These ask the harder questions: what happens when two +/// messages are byte-for-byte identical, and what happens when many arrive at once — the two cases +/// where a consumer quietly loses data instead of failing loudly. +/// +[Collection("Bitween")] +public class RabbitBusAdapterTests +{ + private readonly BitweenFixture _fixture; + + public RabbitBusAdapterTests(BitweenFixture fixture) => _fixture = fixture; + + /// + /// Two identical messages are two messages. + /// + /// Deduplication needs a key, and a publisher that sets no MessageId gives us none. Hashing the + /// body to manufacture one looks like a reasonable fallback and is not: a customer sending the + /// same instruction twice — reorder the same item, the same daily totals, a heartbeat — would + /// have the second silently swallowed for the whole deduplication window, which is thirty days + /// by default. No error, no Xchange, no way to notice. + /// + /// So an unkeyed delivery is handled once per delivery. That trades a possible reprocess on + /// redelivery for never dropping a real message, which is the right way round. + /// + [Fact] + public async Task Two_identical_messages_with_no_message_id_are_two_Xchanges() + { + var queue = Unique("rb-ident"); + var (_, documentId, lease) = await ArrangeAsync(queue); + await using var _ = lease; + + const string body = """{"instruction":"reorder","sku":"A-1"}"""; + + PublishRaw(queue, body, messageId: null); + PublishRaw(queue, body, messageId: null); + + await WaitAsync(async () => await CountAsync(documentId) >= 2, TimeSpan.FromSeconds(45), + "the second identical message was swallowed — a content hash is being used as identity"); + + Assert.Equal(2, await CountAsync(documentId)); + } + + /// + /// The same body WITH message ids is still two messages, and a redelivery of one of them is + /// not. This is the line the previous test is defending: identity comes from the broker's id, + /// never from the payload. + /// + [Fact] + public async Task Identity_comes_from_the_message_id_not_from_the_body() + { + var queue = Unique("rb-identity"); + var (_, documentId, lease) = await ArrangeAsync(queue); + await using var _ = lease; + + const string body = """{"same":"payload"}"""; + var redelivered = Guid.NewGuid().ToString("N"); + + PublishRaw(queue, body, Guid.NewGuid().ToString("N")); + PublishRaw(queue, body, redelivered); + PublishRaw(queue, body, redelivered); // the redelivery + + await WaitAsync(async () => await CountAsync(documentId) >= 2, TimeSpan.FromSeconds(45), + "two distinct messages did not both produce an Xchange"); + + // Settle, then prove the third did NOT land. Waiting for an absence needs a pause; without + // one this asserts on a message still in flight rather than on one that was rejected. + await Task.Delay(TimeSpan.FromSeconds(5)); + Assert.Equal(2, await CountAsync(documentId)); + } + + /// + /// Concurrent deliveries, which is the normal state of a consumer with prefetch above one. + /// + /// Every delivery is handled on the thread pool, so acknowledgements land on the shared channel + /// from several threads at once — and RabbitMQ.Client does not support concurrent application + /// operations on one model. A corrupted frame stream drops the channel and redelivers + /// everything in flight, so what this asserts is the outcome that matters: every message + /// arrives, exactly once, and the connection is still up afterwards. + /// + /// This is an integrity test, not a race detector — a race that is not serialised may still + /// pass on a quiet machine. It fails loudly when the ordering breaks, which is what a test can + /// honestly offer here. + /// + [Fact] + public async Task Every_message_survives_concurrent_delivery_exactly_once() + { + const int count = 60; + + var queue = Unique("rb-concurrent"); + var (dataSourceId, documentId, lease) = await ArrangeAsync(queue); + await using var _ = lease; + + PublishMany(queue, count); + + await WaitAsync(async () => await CountAsync(documentId) >= count, TimeSpan.FromSeconds(90), + $"only {await CountAsync(documentId)} of {count} messages became an Xchange"); + + // Nothing extra either: a channel that dropped mid-flight would redeliver, and with a + // dedupe key per message that would show up as a shortfall rather than a surplus — so + // check both ends. + await Task.Delay(TimeSpan.FromSeconds(5)); + Assert.Equal(count, await CountAsync(documentId)); + + var health = InstanceOf(dataSourceId); + Assert.NotNull(health); + Assert.True(health!.Connected, $"the adapter lost its connection: {health.LastError}"); + + Assert.Equal(count.ToString(), health.Details["received"]); + Assert.Equal(count.ToString(), health.Details["acked"]); + Assert.Equal("0", health.Details["nacked"]); + Assert.Equal("0", health.Details["failed"]); + } + + /// + /// Egress under concurrency. Publishing builds the message properties on the channel too, so a + /// publish that only guards the send is still two channel operations racing. + /// + [Fact] + public async Task Concurrent_publishes_all_reach_the_broker() + { + const int count = 40; + + var queue = Unique("rb-egress"); + DeclareQueue(queue); + + var (_, _, lease) = await ArrangeAsync(Unique("rb-egress-in")); + await using var _ = lease; + + await Task.WhenAll(Enumerable.Range(0, count).Select(i => + lease.Instance.InvokeAsync("Publish", new + { + Endpoint = queue, + Body = $$"""{"n":{{i}}}""", + ContentType = "application/json" + }, timeoutSeconds: 30))); + + await WaitAsync(() => Task.FromResult(Depth(queue) >= count), TimeSpan.FromSeconds(30), + $"only {Depth(queue)} of {count} published messages reached the queue"); + + Assert.Equal((uint)count, Depth(queue)); + } + + // ---------------------------------------------------------------- helpers + + private static string Unique(string prefix) => $"{prefix}-{Guid.NewGuid():N}"[..24]; + + private async Task<(int DataSourceId, int DocumentId, AdapterLease Lease)> ArrangeAsync(string queue) + { + int dataSourceId, documentId; + + await using (var scope = _fixture.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + + var dataSource = new DataSource + { + Name = Unique("ds"), + AdapterId = BusAdapters.RabbitMq, + Kind = DataSourceKind.Broker, + Properties = new Dictionary(_fixture.ExternalRabbitProperties) + { + ["Endpoints"] = queue + } + }; + db.Add(dataSource); + + var document = new Document(null, Unique("doc"), DocumentFormat.Json); + db.Add(document); + await db.SaveChangesAsync(); + + db.Add(new BusGateway + { + Name = Unique("gw"), + DocumentId = document.Id, + DataSourceId = dataSource.Id, + Endpoint = queue + }); + await db.SaveChangesAsync(); + + // The infolink cache is a ten-minute singleton snapshot; without revoking it a + // brand-new Document resolves to null and the message is acked and silently dropped. + scope.ServiceProvider.GetRequiredService().Revoke(); + + dataSourceId = dataSource.Id; + documentId = document.Id; + } + + var host = _fixture.App.Services.GetRequiredService(); + var instance = await host.StartExclusiveAsync(new AdapterSpec + { + AdapterId = BusAdapters.RabbitMq, + InstanceKey = dataSourceId.ToString(), + StartupValues = new Dictionary(_fixture.ExternalRabbitProperties) + { + ["Endpoints"] = queue + } + }); + + return (dataSourceId, documentId, + new AdapterLease(host, BusAdapters.RabbitMq, dataSourceId.ToString(), instance)); + } + + private sealed class AdapterLease : IAsyncDisposable + { + private readonly IResidentAdapterHost _host; + private readonly string _adapterId; + private readonly string _instanceKey; + + public AdapterLease(IResidentAdapterHost host, string adapterId, string instanceKey, + ResidentAdapterInstance instance) + { + _host = host; + _adapterId = adapterId; + _instanceKey = instanceKey; + Instance = instance; + } + + public ResidentAdapterInstance Instance { get; } + + public ValueTask DisposeAsync() => + new(_host.StopAsync(_adapterId, _instanceKey, drain: false)); + } + + private InstanceHealth InstanceOf(int dataSourceId) => + _fixture.App.Services.GetRequiredService() + .Describe().FirstOrDefault(h => h.InstanceKey == dataSourceId.ToString()); + + private async Task CountAsync(int documentId) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + return await db.Set().AsNoTracking().CountAsync(x => x.DocumentId == documentId); + } + + /// Publishes with full control over the message id, including leaving it unset. + private void PublishRaw(string queue, string body, string messageId) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + + channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false); + + var properties = channel.CreateBasicProperties(); + properties.ContentType = "application/json"; + properties.DeliveryMode = 2; + if (messageId != null) properties.MessageId = messageId; + + channel.BasicPublish("", queue, properties, Encoding.UTF8.GetBytes(body)); + } + + private void PublishMany(string queue, int count) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + + channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false); + + for (var i = 0; i < count; i++) + { + var properties = channel.CreateBasicProperties(); + properties.ContentType = "application/json"; + properties.DeliveryMode = 2; + properties.MessageId = Guid.NewGuid().ToString("N"); + + channel.BasicPublish("", queue, properties, + Encoding.UTF8.GetBytes($$"""{"n":{{i}}}""")); + } + } + + private void DeclareQueue(string queue) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false); + } + + private uint Depth(string queue) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + return channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false).MessageCount; + } + + private IConnection ExternalConnection() => new ConnectionFactory + { + HostName = _fixture.ExternalRabbitHost, + Port = _fixture.ExternalRabbitPort, + UserName = _fixture.ExternalRabbitUser, + Password = _fixture.ExternalRabbitPassword + }.CreateConnection("rabbit-adapter-tests"); + + private static async Task WaitAsync(Func> condition, TimeSpan timeout, string because) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + try { if (await condition()) return; } catch { /* still settling */ } + await Task.Delay(300); + } + Assert.Fail($"Timed out after {timeout}: {because}"); + } +} diff --git a/SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.Designer.cs b/SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.Designer.cs new file mode 100644 index 00000000..70653521 --- /dev/null +++ b/SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.Designer.cs @@ -0,0 +1,2290 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using SW.Bitween.MsSql; + +#nullable disable + +namespace SW.Bitween.MsSql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906103810_ExternalBusDataSources")] + partial class ExternalBusDataSources + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.HasSequence("DocumentIds"); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Deleted") + .HasColumnType("bit"); + + b.Property("Disabled") + .HasColumnType("bit"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("EmailProvider") + .HasColumnType("tinyint"); + + b.Property("FailedLoginCount") + .HasColumnType("int"); + + b.Property("LockoutEnd") + .HasColumnType("datetime2"); + + b.Property("LoginMethods") + .HasColumnType("tinyint"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Role") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique() + .HasFilter("[Email] IS NOT NULL"); + + b.ToTable("Accounts", (string)null); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("RoleId") + .HasColumnType("int"); + + b.HasKey("AccountId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AccountRoles", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("LoginMethod") + .HasColumnType("tinyint"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.ToTable("RefreshTokens", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("IsSystem") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Permissions") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles", (string)null); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime2"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime2"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("SecretProperties") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime2"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("On"); + + b.ToTable("DelayedRetries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValueSql("NEXT VALUE FOR [DocumentIds]"); + + SqlServerPropertyBuilderExtensions.UseSequence(b.Property("Id"), "DocumentIds"); + + b.Property("BusEnabled") + .HasColumnType("bit"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Code") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("bit"); + + b.Property("DocumentFormat") + .HasColumnType("int"); + + b.Property("DuplicateInterval") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("PromotedProperties") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("BusMessageTypeName") + .IsUnique() + .HasFilter("[BusMessageTypeName] IS NOT NULL"); + + b.HasIndex("Code") + .IsUnique() + .HasFilter("[Code] IS NOT NULL"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Documents", (string)null); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("StateAfter") + .HasColumnType("nvarchar(max)"); + + b.Property("StateBefore") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("DocumentId"); + + b.ToTable("DocumentTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("UrlName") + .IsUnique(); + + b.ToTable("ApiGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("int"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("ApiGatewayPartners", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("DocumentId"); + + b.ToTable("BusGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("MatchExpression") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("BusGatewayId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("BusGatewayRoutes", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Values") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("GlobalAdapterValuesSets", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("RunOnBadResult") + .HasColumnType("bit"); + + b.Property("RunOnFailedResult") + .HasColumnType("bit"); + + b.Property("RunOnSubscriptions") + .HasColumnType("nvarchar(max)"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.ToTable("Notifiers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("bit"); + + b.Property("Data") + .HasColumnType("nvarchar(max)"); + + b.Property("FileName") + .HasColumnType("nvarchar(max)"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("OnHoldXchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("Partners", (string)null); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ExchangeIds") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("Outcome") + .HasColumnType("int"); + + b.Property("StartedOn") + .HasColumnType("datetime2"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId", "StartedOn"); + + b.ToTable("ReceiveAttempts", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("AlertMode") + .HasColumnType("tinyint"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryAlertOverrides", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("AttemptsUsed") + .HasColumnType("int"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("datetime2"); + + b.Property("LastAttemptOn") + .HasColumnType("datetime2"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryGroupUsages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Groups") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.ToTable("RetryPolicies", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Value") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Settings", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("datetime2"); + + b.Property("AggregationForId") + .HasColumnType("int"); + + b.Property("AggregationTarget") + .HasColumnType("tinyint"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CustomRetryPolicy") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentFilter") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("IsRunning") + .HasColumnType("bit"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("MatchExpression") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("PausedOn") + .HasColumnType("datetime2"); + + b.Property("ReceiveOn") + .HasColumnType("datetime2"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ReceiverProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryPolicyId") + .HasColumnType("int"); + + b.Property("Temporary") + .HasColumnType("bit"); + + b.Property("Type") + .HasColumnType("tinyint"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ValidatorProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("WorkGroupId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AggregationForId"); + + b.HasIndex("CategoryId"); + + b.HasIndex("DocumentId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("ResponseSubscriptionId"); + + b.HasIndex("RetryPolicyId"); + + b.HasIndex("WorkGroupId"); + + b.ToTable("Subscriptions", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("nvarchar(450)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique() + .HasFilter("[Code] IS NOT NULL"); + + b.ToTable("SubscriptionCategory"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("StateAfter") + .HasColumnType("nvarchar(max)"); + + b.Property("StateBefore") + .HasColumnType("nvarchar(max)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("SubscriptionTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Options") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("CorrelationId") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("InputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("InputSize") + .HasColumnType("int"); + + b.Property("ManualRetry") + .HasColumnType("bit"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryFor") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("StartedOn") + .HasColumnType("datetime2"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId"); + + b.HasIndex("InputHash"); + + b.HasIndex("RetryFor"); + + b.HasIndex("StartedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("Xchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AggregatedOn") + .HasColumnType("datetime2"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("AggregationXchangeId"); + + b.ToTable("XchangeAggregations", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DeliveredOn") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeliveredOn"); + + b.ToTable("XchangeDeliveries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("NotifierId") + .HasColumnType("int"); + + b.Property("NotifierName") + .HasColumnType("nvarchar(max)"); + + b.Property("Success") + .HasColumnType("bit"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.ToTable("XchangeNotifications", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("Hits") + .HasMaxLength(2000) + .IsUnicode(false) + .HasColumnType("varchar(2000)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("PropertiesRaw") + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("PropertiesRaw"); + + b.ToTable("XchangePromotedProperties", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AttemptNumber") + .HasColumnType("int"); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("OutputBad") + .HasColumnType("bit"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("OutputHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OutputSize") + .HasColumnType("int"); + + b.Property("ResponseBad") + .HasColumnType("bit"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ResponseSize") + .HasColumnType("int"); + + b.Property("ResponseXchangeId") + .HasColumnType("nvarchar(max)"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("RetryGroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("Success") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.HasIndex("RetryGroupId"); + + b.ToTable("XchangeResults", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("bit"); + + b.ToTable((string)null); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("nvarchar(max)") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("datetime2") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("nvarchar(max)") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("datetime2") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("bit") + .HasColumnName("success"); + + b.HasKey("Id"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("varbinary(max)") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_blob_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("nvarchar(450)") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("varbinary(max)") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("QRTZ_calendars", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("nvarchar(450)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_cron_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("nvarchar(450)") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bit") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bit") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_QRTZ_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_QRTZ_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_QRTZ_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_QRTZ_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_QRTZ_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_nm_gp"); + + b.ToTable("QRTZ_fired_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("nvarchar(450)") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bit") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bit") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bit") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("varbinary(max)") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bit") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("QRTZ_job_details", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("nvarchar(450)") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("QRTZ_locks", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("QRTZ_paused_trigger_grps", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("nvarchar(450)") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("QRTZ_scheduler_state", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bit") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bit") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("int") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("int") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("nvarchar(450)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simprop_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simple_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("nvarchar(450)") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("nvarchar(450)") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("varbinary(max)") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("int") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("QRTZ_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b1.HasKey("PartnerId", "Id"); + + b1.HasIndex("Key") + .IsUnique(); + + b1.ToTable("PartnerApiCredentials", (string)null); + + b1.WithOwner() + .HasForeignKey("PartnerId"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_AggFor"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_RespSub"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("bit"); + + b1.Property("On") + .HasColumnType("bigint"); + + b1.Property("Recurrence") + .HasColumnType("tinyint"); + + b1.HasKey("SubscriptionId", "Id"); + + b1.ToTable("SubscriptionSchedules", (string)null); + + b1.WithOwner() + .HasForeignKey("SubscriptionId"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.cs b/SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.cs new file mode 100644 index 00000000..13ce3da6 --- /dev/null +++ b/SW.Bitween.MsSql/Migrations/20260906103810_ExternalBusDataSources.cs @@ -0,0 +1,159 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.MsSql.Migrations +{ + /// + public partial class ExternalBusDataSources : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "DataSourceId", + table: "BusGateways", + type: "int", + nullable: true); + + migrationBuilder.AddColumn( + name: "Endpoint", + table: "BusGateways", + type: "varchar(500)", + unicode: false, + maxLength: 500, + nullable: true); + + migrationBuilder.AddColumn( + name: "EndpointProperties", + table: "BusGateways", + type: "nvarchar(max)", + nullable: true); + + migrationBuilder.CreateTable( + name: "ClusterLeases", + columns: table => new + { + Id = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: false), + Term = table.Column(type: "bigint", nullable: false), + OwnerNode = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: true), + AcquiredOn = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ClusterLeases", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "DataSources", + columns: table => new + { + Id = table.Column(type: "int", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + Name = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: false), + AdapterId = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: false), + Kind = table.Column(type: "int", nullable: false), + Properties = table.Column(type: "nvarchar(max)", nullable: true), + SecretProperties = table.Column(type: "nvarchar(max)", nullable: true), + Inactive = table.Column(type: "bit", nullable: false), + DeduplicationWindowDays = table.Column(type: "int", nullable: false), + LastKnownState = table.Column(type: "varchar(100)", unicode: false, maxLength: 100, nullable: true), + LastHeartbeatOn = table.Column(type: "datetime2", nullable: true), + LastException = table.Column(type: "nvarchar(max)", nullable: true), + ConsecutiveFailures = table.Column(type: "int", nullable: false), + OwnedByNode = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: true), + CreatedOn = table.Column(type: "datetime2", nullable: false), + CreatedBy = table.Column(type: "nvarchar(max)", nullable: true), + ModifiedOn = table.Column(type: "datetime2", nullable: true), + ModifiedBy = table.Column(type: "nvarchar(max)", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_DataSources", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "InboundMessages", + columns: table => new + { + Id = table.Column(type: "varchar(400)", unicode: false, maxLength: 400, nullable: false), + DataSourceId = table.Column(type: "int", nullable: false), + XchangeId = table.Column(type: "varchar(50)", unicode: false, maxLength: 50, nullable: true), + SeenOn = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_InboundMessages", x => x.Id); + table.ForeignKey( + name: "FK_InboundMessages_DataSources_DataSourceId", + column: x => x.DataSourceId, + principalTable: "DataSources", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_BusGateways_DataSourceId", + table: "BusGateways", + column: "DataSourceId"); + + migrationBuilder.CreateIndex( + name: "IX_DataSources_Name", + table: "DataSources", + column: "Name", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_InboundMessages_DataSourceId", + table: "InboundMessages", + column: "DataSourceId"); + + migrationBuilder.CreateIndex( + name: "IX_InboundMessages_SeenOn", + table: "InboundMessages", + column: "SeenOn"); + + migrationBuilder.AddForeignKey( + name: "FK_BusGateways_DataSources_DataSourceId", + table: "BusGateways", + column: "DataSourceId", + principalTable: "DataSources", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_BusGateways_DataSources_DataSourceId", + table: "BusGateways"); + + migrationBuilder.DropTable( + name: "ClusterLeases"); + + migrationBuilder.DropTable( + name: "InboundMessages"); + + migrationBuilder.DropTable( + name: "DataSources"); + + migrationBuilder.DropIndex( + name: "IX_BusGateways_DataSourceId", + table: "BusGateways"); + + migrationBuilder.DropColumn( + name: "DataSourceId", + table: "BusGateways"); + + migrationBuilder.DropColumn( + name: "Endpoint", + table: "BusGateways"); + + migrationBuilder.DropColumn( + name: "EndpointProperties", + table: "BusGateways"); + } + } +} diff --git a/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs b/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs index 4af58e1a..dde31f34 100644 --- a/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs +++ b/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs @@ -215,6 +215,129 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime2"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime2"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("SecretProperties") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime2"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => { b.Property("Id") @@ -419,9 +542,20 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("CreatedOn") .HasColumnType("datetime2"); + b.Property("DataSourceId") + .HasColumnType("int"); + b.Property("DocumentId") .HasColumnType("int"); + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("nvarchar(max)"); + b.Property("Inactive") .HasColumnType("bit"); @@ -438,6 +572,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); + b.HasIndex("DataSourceId"); + b.HasIndex("DocumentId"); b.ToTable("BusGateways", (string)null); @@ -1793,6 +1929,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired(); }); + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => { b.HasOne("SW.Bitween.Domain.Document", "Document") @@ -1833,11 +1978,18 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + b.HasOne("SW.Bitween.Domain.Document", null) .WithMany() .HasForeignKey("DocumentId") .OnDelete(DeleteBehavior.Restrict) .IsRequired(); + + b.Navigation("DataSource"); }); modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => diff --git a/SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.Designer.cs b/SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.Designer.cs new file mode 100644 index 00000000..cf74527e --- /dev/null +++ b/SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.Designer.cs @@ -0,0 +1,2283 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using SW.Bitween.MySql; + +#nullable disable + +namespace SW.Bitween.MySql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906103806_ExternalBusDataSources")] + partial class ExternalBusDataSources + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + MySqlModelBuilderExtensions.AutoIncrementColumns(modelBuilder); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Deleted") + .HasColumnType("tinyint(1)"); + + b.Property("Disabled") + .HasColumnType("tinyint(1)"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("EmailProvider") + .HasColumnType("tinyint unsigned"); + + b.Property("FailedLoginCount") + .HasColumnType("int"); + + b.Property("LockoutEnd") + .HasColumnType("datetime(6)"); + + b.Property("LoginMethods") + .HasColumnType("tinyint unsigned"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Role") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique(); + + b.ToTable("Accounts", (string)null); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("RoleId") + .HasColumnType("int"); + + b.HasKey("AccountId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AccountRoles", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("LoginMethod") + .HasColumnType("tinyint unsigned"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.ToTable("RefreshTokens", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b.Property("IsSystem") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("Permissions") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles", (string)null); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime(6)"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime(6)"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("SecretProperties") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime(6)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("On") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("On"); + + b.ToTable("DelayedRetries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusEnabled") + .HasColumnType("tinyint(1)"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Code") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("tinyint(1)"); + + b.Property("DocumentFormat") + .HasColumnType("int"); + + b.Property("DuplicateInterval") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("PromotedProperties") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("BusMessageTypeName") + .IsUnique(); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Documents", (string)null); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("StateAfter") + .HasColumnType("longtext"); + + b.Property("StateBefore") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("DocumentId"); + + b.ToTable("DocumentTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("UrlName") + .IsUnique(); + + b.ToTable("ApiGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("int"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("ApiGatewayPartners", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("DocumentId"); + + b.ToTable("BusGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("MatchExpression") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("BusGatewayId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("BusGatewayRoutes", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Name") + .HasColumnType("longtext"); + + b.Property("Values") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("GlobalAdapterValuesSets", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("RunOnBadResult") + .HasColumnType("tinyint(1)"); + + b.Property("RunOnFailedResult") + .HasColumnType("tinyint(1)"); + + b.Property("RunOnSubscriptions") + .HasColumnType("longtext"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.ToTable("Notifiers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("tinyint(1)"); + + b.Property("Data") + .HasColumnType("longtext"); + + b.Property("FileName") + .HasColumnType("longtext"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("varchar(1024)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("OnHoldXchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterProperties") + .HasColumnType("longtext"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("Partners", (string)null); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasMaxLength(4000) + .HasColumnType("varchar(4000)"); + + b.Property("ExchangeIds") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("Outcome") + .HasColumnType("int"); + + b.Property("StartedOn") + .HasColumnType("datetime(6)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId", "StartedOn"); + + b.ToTable("ReceiveAttempts", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("char(36)"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("longtext"); + + b.Property("AlertMode") + .HasColumnType("tinyint unsigned"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryAlertOverrides", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("char(36)"); + + b.Property("AttemptsUsed") + .HasColumnType("int"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("LastAttemptOn") + .HasColumnType("datetime(6)"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryGroupUsages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("longtext"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Groups") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("RetryPolicies", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Value") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("Settings", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("datetime(6)"); + + b.Property("AggregationForId") + .HasColumnType("int"); + + b.Property("AggregationTarget") + .HasColumnType("tinyint unsigned"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CustomRetryPolicy") + .HasColumnType("longtext"); + + b.Property("DocumentFilter") + .HasColumnType("longtext"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("IsRunning") + .HasColumnType("tinyint(1)"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("longtext"); + + b.Property("MatchExpression") + .HasColumnType("longtext"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("PausedOn") + .HasColumnType("datetime(6)"); + + b.Property("ReceiveOn") + .HasColumnType("datetime(6)"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ReceiverProperties") + .HasColumnType("longtext"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryPolicyId") + .HasColumnType("int"); + + b.Property("Temporary") + .HasColumnType("tinyint(1)"); + + b.Property("Type") + .HasColumnType("tinyint unsigned"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ValidatorProperties") + .HasColumnType("longtext"); + + b.Property("WorkGroupId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AggregationForId"); + + b.HasIndex("CategoryId"); + + b.HasIndex("DocumentId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("ResponseSubscriptionId"); + + b.HasIndex("RetryPolicyId"); + + b.HasIndex("WorkGroupId"); + + b.ToTable("Subscriptions", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("varchar(255)"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Description") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("SubscriptionCategory"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("StateAfter") + .HasColumnType("longtext"); + + b.Property("StateBefore") + .HasColumnType("longtext"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("SubscriptionTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("Name") + .HasColumnType("longtext"); + + b.Property("Options") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("CorrelationId") + .HasColumnType("longtext"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("InputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("InputSize") + .HasColumnType("int"); + + b.Property("ManualRetry") + .HasColumnType("tinyint(1)"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("longtext"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("varchar(1024)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryFor") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("StartedOn") + .HasColumnType("datetime(6)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId"); + + b.HasIndex("InputHash"); + + b.HasIndex("RetryFor"); + + b.HasIndex("StartedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("Xchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AggregatedOn") + .HasColumnType("datetime(6)"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("AggregationXchangeId"); + + b.ToTable("XchangeAggregations", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DeliveredOn") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("DeliveredOn"); + + b.ToTable("XchangeDeliveries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("NotifierId") + .HasColumnType("int"); + + b.Property("NotifierName") + .HasColumnType("longtext"); + + b.Property("Success") + .HasColumnType("tinyint(1)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.ToTable("XchangeNotifications", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("Hits") + .HasMaxLength(2000) + .IsUnicode(false) + .HasColumnType("varchar(2000)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("PropertiesRaw") + .HasColumnType("varchar(255)"); + + b.HasKey("Id"); + + b.HasIndex("PropertiesRaw"); + + b.ToTable("XchangePromotedProperties", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AttemptNumber") + .HasColumnType("int"); + + b.Property("Exception") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("OutputBad") + .HasColumnType("tinyint(1)"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("OutputHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OutputSize") + .HasColumnType("int"); + + b.Property("ResponseBad") + .HasColumnType("tinyint(1)"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("ResponseSize") + .HasColumnType("int"); + + b.Property("ResponseXchangeId") + .HasColumnType("longtext"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b.Property("RetryGroupId") + .HasColumnType("char(36)"); + + b.Property("Success") + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.HasIndex("RetryGroupId"); + + b.ToTable("XchangeResults", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("tinyint(1)"); + + b.ToTable((string)null); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("longtext") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("datetime(6)") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("longtext") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("datetime(6)") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("tinyint(1)") + .HasColumnName("success"); + + b.HasKey("Id"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("longblob") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_blob_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("varchar(200)") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("longblob") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("QRTZ_calendars", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("varchar(200)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_cron_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("varchar(200)") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("tinyint(1)") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("tinyint(1)") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_QRTZ_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_QRTZ_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_QRTZ_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_QRTZ_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_QRTZ_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_nm_gp"); + + b.ToTable("QRTZ_fired_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("varchar(200)") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("tinyint(1)") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("tinyint(1)") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("tinyint(1)") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("longblob") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("tinyint(1)") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("QRTZ_job_details", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("varchar(200)") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("QRTZ_locks", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("QRTZ_paused_trigger_grps", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("varchar(200)") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("QRTZ_scheduler_state", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("tinyint(1)") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("tinyint(1)") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("int") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("int") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("varchar(200)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simprop_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simple_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("varchar(200)") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("varchar(200)") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("longblob") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("int") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("QRTZ_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b1.HasKey("PartnerId", "Id"); + + b1.HasIndex("Key") + .IsUnique(); + + b1.ToTable("PartnerApiCredentials", (string)null); + + b1.WithOwner() + .HasForeignKey("PartnerId"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_AggFor"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_RespSub"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("tinyint(1)"); + + b1.Property("On") + .HasColumnType("bigint"); + + b1.Property("Recurrence") + .HasColumnType("tinyint unsigned"); + + b1.HasKey("SubscriptionId", "Id"); + + b1.ToTable("SubscriptionSchedules", (string)null); + + b1.WithOwner() + .HasForeignKey("SubscriptionId"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.cs b/SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.cs new file mode 100644 index 00000000..13f7f958 --- /dev/null +++ b/SW.Bitween.MySql/Migrations/20260906103806_ExternalBusDataSources.cs @@ -0,0 +1,178 @@ +using System; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.MySql.Migrations +{ + /// + public partial class ExternalBusDataSources : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "DataSourceId", + table: "BusGateways", + type: "int", + nullable: true); + + migrationBuilder.AddColumn( + name: "Endpoint", + table: "BusGateways", + type: "varchar(500)", + unicode: false, + maxLength: 500, + nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.AddColumn( + name: "EndpointProperties", + table: "BusGateways", + type: "longtext", + nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "ClusterLeases", + columns: table => new + { + Id = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + Term = table.Column(type: "bigint", nullable: false), + OwnerNode = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + AcquiredOn = table.Column(type: "datetime(6)", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ClusterLeases", x => x.Id); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "DataSources", + columns: table => new + { + Id = table.Column(type: "int", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + Name = table.Column(type: "varchar(200)", maxLength: 200, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + AdapterId = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + Kind = table.Column(type: "int", nullable: false), + Properties = table.Column(type: "longtext", nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + SecretProperties = table.Column(type: "longtext", nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + Inactive = table.Column(type: "tinyint(1)", nullable: false), + DeduplicationWindowDays = table.Column(type: "int", nullable: false), + LastKnownState = table.Column(type: "varchar(100)", unicode: false, maxLength: 100, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + LastHeartbeatOn = table.Column(type: "datetime(6)", nullable: true), + LastException = table.Column(type: "longtext", nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + ConsecutiveFailures = table.Column(type: "int", nullable: false), + OwnedByNode = table.Column(type: "varchar(200)", unicode: false, maxLength: 200, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + CreatedOn = table.Column(type: "datetime(6)", nullable: false), + CreatedBy = table.Column(type: "longtext", nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + ModifiedOn = table.Column(type: "datetime(6)", nullable: true), + ModifiedBy = table.Column(type: "longtext", nullable: true) + .Annotation("MySql:CharSet", "utf8mb4") + }, + constraints: table => + { + table.PrimaryKey("PK_DataSources", x => x.Id); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "InboundMessages", + columns: table => new + { + Id = table.Column(type: "varchar(400)", unicode: false, maxLength: 400, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + DataSourceId = table.Column(type: "int", nullable: false), + XchangeId = table.Column(type: "varchar(50)", unicode: false, maxLength: 50, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + SeenOn = table.Column(type: "datetime(6)", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_InboundMessages", x => x.Id); + table.ForeignKey( + name: "FK_InboundMessages_DataSources_DataSourceId", + column: x => x.DataSourceId, + principalTable: "DataSources", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateIndex( + name: "IX_BusGateways_DataSourceId", + table: "BusGateways", + column: "DataSourceId"); + + migrationBuilder.CreateIndex( + name: "IX_DataSources_Name", + table: "DataSources", + column: "Name", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_InboundMessages_DataSourceId", + table: "InboundMessages", + column: "DataSourceId"); + + migrationBuilder.CreateIndex( + name: "IX_InboundMessages_SeenOn", + table: "InboundMessages", + column: "SeenOn"); + + migrationBuilder.AddForeignKey( + name: "FK_BusGateways_DataSources_DataSourceId", + table: "BusGateways", + column: "DataSourceId", + principalTable: "DataSources", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_BusGateways_DataSources_DataSourceId", + table: "BusGateways"); + + migrationBuilder.DropTable( + name: "ClusterLeases"); + + migrationBuilder.DropTable( + name: "InboundMessages"); + + migrationBuilder.DropTable( + name: "DataSources"); + + migrationBuilder.DropIndex( + name: "IX_BusGateways_DataSourceId", + table: "BusGateways"); + + migrationBuilder.DropColumn( + name: "DataSourceId", + table: "BusGateways"); + + migrationBuilder.DropColumn( + name: "Endpoint", + table: "BusGateways"); + + migrationBuilder.DropColumn( + name: "EndpointProperties", + table: "BusGateways"); + } + } +} diff --git a/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs b/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs index 113a9d3d..bc85f746 100644 --- a/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs +++ b/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs @@ -212,6 +212,129 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime(6)"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime(6)"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("SecretProperties") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime(6)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => { b.Property("Id") @@ -413,9 +536,20 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("CreatedOn") .HasColumnType("datetime(6)"); + b.Property("DataSourceId") + .HasColumnType("int"); + b.Property("DocumentId") .HasColumnType("int"); + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("longtext"); + b.Property("Inactive") .HasColumnType("tinyint(1)"); @@ -432,6 +566,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); + b.HasIndex("DataSourceId"); + b.HasIndex("DocumentId"); b.ToTable("BusGateways", (string)null); @@ -1786,6 +1922,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired(); }); + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => { b.HasOne("SW.Bitween.Domain.Document", "Document") @@ -1826,11 +1971,18 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + b.HasOne("SW.Bitween.Domain.Document", null) .WithMany() .HasForeignKey("DocumentId") .OnDelete(DeleteBehavior.Restrict) .IsRequired(); + + b.Navigation("DataSource"); }); modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index 29a15fd7..f666eb32 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -162,9 +162,10 @@ public void ConfigureServices(IServiceCollection services) configure.AdapterRemotePath = bitweenOptions.AdapterPath; }); - // External bus providers. Off by default: a broker connection is exclusive, and - // placement across nodes is not implemented yet, so every instance would otherwise - // try to hold the same connection. Turn it on only where a single instance owns them. + // External bus providers. Off by default because it is opt-in, not because it is + // unsafe to run on more than one node: a broker connection is exclusive, and every + // data source is held through a lease with a database-issued fencing term, so only + // one node consumes any given source. See BusProviderSupervisor and ILeaderElection. if (bitweenOptions.BusProvidersEnabled) { services.AddResidentAdapters(configure => From 3ca63d10c49bafde8cd3c118d6059cd11d71f5bf Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Sun, 6 Sep 2026 15:13:55 +0300 Subject: [PATCH 10/43] feat: configure and observe external bus gateways from the UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The external bus feature had no API and no UI: it was reachable only by writing rows into the database, which is how every one of its tests set itself up. This adds both, in SW.Bitween.Web/ClientApp — the UI that ships inside the API — and the multi-integration tests the single-queue suite never covered. 217 unit + 258 integration + 90 ClientApp tests pass. API - /datasources: search, get, create, update, delete, plus test and telemetry. - Secrets are masked on the way out and merged back on save, so editing the prefetch cannot overwrite the password with a row of dots, and a data source copied out of a response cannot authenticate with the sentinel. A property NAMED like a credential is masked whether or not anyone declared it — the one nobody ticks is the one that ends up in a JSON response. - The list carries no connection settings at all. Even masked, sending every credential to render a table row is exposure with no purpose. - test runs the real adapter against the stored settings and reports stage by stage, but starts it with Consume=false so the customer's queues are inspected rather than drained. Without it the only way to find a wrong password is to save, wait out a reconcile, and read the health column. - BusGateway gained DataSourceId and Endpoint. An external gateway with no endpoint is refused — it would sit connected and never receive anything — and so is a second gateway on an endpoint another already reads, which would make every message a coin toss between them. Observability - /datasources/{id}/telemetry reads the heartbeat rather than the row, which only carries what the last reconcile wrote back. The panel keeps the two sources apart on purpose: adapter-reported figures (per-queue backlog, received/acked/nacked/failed) stop arriving the moment the adapter wedges, while host-observed ones (pid, memory, CPU, threads, uptime, restarts) need no cooperation and still answer. That is what separates "the broker is quiet" from "the adapter is stuck". - It is scoped to the node that answers and says so, because a broker connection is exclusive and no other node can see its figures. UI - A Data sources area: list with connection health and which node holds each, a create page, and a detail page with settings, the live panel and Test. - On the bus gateway, a Source control — internal bus or a broker, with the endpoint and that connection's health. A dialog rather than a canvas node: the canvas draws what happens to a message per route, and where messages come from belongs to the gateway. - Fixed two bugs the new column exposed: "Type not on bus" was shown for external gateways in both the list and the toolbar. It is meaningless there — an external gateway never touches Bitween's own bus — and it was hiding the gateway's real problem, which was that it had no routes. Adapters - Both gained Consume=false: connect and declare, but do not consume. This is what makes a connection test safe to run against a live queue. Tests - DataSourceApiTests (14): secrets never returned in clear, an undeclared credential masked anyway, a masked secret surviving an unrelated edit, a sentinel with nothing behind it dropped, and the gateway guards. - SharedBrokerTests (8): one Acme broker, five queues, three of them Bitween's. One connection and one process serve all three gateways; thirty interleaved messages each land on their own information type; a queue nobody pointed Bitween at stays untouched; a fourth integration joins without disturbing the others; deactivating one gateway stops only its queue and leaves its messages on the broker rather than acking them away; the same message id on two queues is two messages; two data sources on one broker are owned separately; and each queue runs its own subscription and records its own result. - The dedupe-scoping claim is mutation-verified: dropping the endpoint from the key makes that test fail. Also corrects CLAUDE.md, which documented the standalone Bitween-UI repo as the dashboard and never mentioned ClientApp at all. Co-Authored-By: Claude Opus 5 --- .../RabbitBusHandler.cs | 10 + .../RabbitOptions.cs | 6 + SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs | 10 +- SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs | 6 + .../Resources/BusGateways/Create.cs | 36 +- SW.Bitween.Api/Resources/BusGateways/Get.cs | 13 + .../Resources/BusGateways/Search.cs | 11 + .../Resources/BusGateways/Update.cs | 34 +- .../Resources/DataSources/Create.cs | 67 ++ .../Resources/DataSources/Delete.cs | 46 ++ SW.Bitween.Api/Resources/DataSources/Get.cs | 57 ++ .../Resources/DataSources/Search.cs | 67 ++ .../Resources/DataSources/Secrets.cs | 89 +++ .../Resources/DataSources/Telemetry.cs | 92 +++ SW.Bitween.Api/Resources/DataSources/Test.cs | 129 ++++ .../Resources/DataSources/Update.cs | 62 ++ .../Tests/DataSourceApiTests.cs | 432 ++++++++++++ .../Tests/SharedBrokerTests.cs | 630 ++++++++++++++++++ SW.Bitween.Sdk/Model/BusGateway.cs | 26 + SW.Bitween.Sdk/Model/DataSource.cs | 146 ++++ SW.Bitween.Sdk/Model/Permissions.cs | 19 + SW.Bitween.Web/ClientApp/src/api/client.ts | 45 +- .../ClientApp/src/api/http/dataSources.ts | 158 +++++ .../ClientApp/src/api/http/gateways.ts | 51 +- .../ClientApp/src/api/http/httpClient.ts | 2 + SW.Bitween.Web/ClientApp/src/api/queryKeys.ts | 9 + SW.Bitween.Web/ClientApp/src/api/types.ts | 102 +++ SW.Bitween.Web/ClientApp/src/nav.ts | 4 + .../src/pages/bus-gateways/BusGatewayPage.tsx | 46 +- .../pages/bus-gateways/BusGatewaysPage.tsx | 24 +- .../src/pages/bus-gateways/SourceDialog.tsx | 165 +++++ .../pages/data-sources/ConnectionBadge.tsx | 40 ++ .../pages/data-sources/DataSourceNewPage.tsx | 90 +++ .../src/pages/data-sources/DataSourcePage.tsx | 393 +++++++++++ .../pages/data-sources/DataSourcesPage.tsx | 177 +++++ .../src/pages/data-sources/LiveConnection.tsx | 171 +++++ .../src/pages/data-sources/providers.ts | 78 +++ SW.Bitween.Web/ClientApp/src/router.tsx | 28 + 38 files changed, 3558 insertions(+), 13 deletions(-) create mode 100644 SW.Bitween.Api/Resources/DataSources/Create.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Delete.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Get.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Search.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Secrets.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Telemetry.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Test.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Update.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/SharedBrokerTests.cs create mode 100644 SW.Bitween.Sdk/Model/DataSource.cs create mode 100644 SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts create mode 100644 SW.Bitween.Web/ClientApp/src/pages/bus-gateways/SourceDialog.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/ConnectionBadge.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourceNewPage.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcesPage.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/LiveConnection.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/providers.ts diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs index 1a2329f5..dd01267b 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs @@ -93,6 +93,16 @@ public Task StartAsync(IAdapterContext context, CancellationToken cancellationTo DeclareTopology(_consumeChannel); + // Manage-only: the connection is up and the topology is declared, but nothing is consumed. + // TestConnection still checks every endpoint, because it reads _endpoints rather than the + // consumers — so a test proves the queues exist without draining them. + if (!_options.Consume) + { + _state = "Idle"; + _logger.LogInformation("Connected to {Host} without consuming (Consume=false).", _options.Host); + return Task.CompletedTask; + } + foreach (var endpoint in _endpoints) { var consumer = new EventingBasicConsumer(_consumeChannel); diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs index db2120f8..8343403f 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs @@ -29,6 +29,12 @@ public class RabbitOptions public string ExchangeType { get; set; } = "topic"; public string RoutingKey { get; set; } + /// + /// False connects and declares but does not consume. This is what a connection test runs as: + /// without it, testing a data source would start pulling messages off the customer's queue. + /// + public bool Consume { get; set; } = true; + /// Broker-side backpressure; pairs with the host's credit window. public ushort Prefetch { get; set; } = 16; diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs index 873187b7..dbda538f 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs @@ -75,9 +75,13 @@ public Task StartAsync(IAdapterContext context, CancellationToken cancellationTo _options.Region, _endpoints.Count, _options.WaitTimeSeconds); // One poller per queue. Long polling means these are cheap: a blocked receive costs - // nothing until a message arrives or the wait expires. - foreach (var endpoint in _endpoints) - _pollers.Add(Task.Run(() => PollAsync(endpoint, _stopping.Token))); + // nothing until a message arrives or the wait expires. Skipped entirely for a connection + // test, which still checks every endpoint because TestConnection reads _endpoints. + if (_options.Consume) + foreach (var endpoint in _endpoints) + _pollers.Add(Task.Run(() => PollAsync(endpoint, _stopping.Token))); + else + _state = "Idle"; return Task.CompletedTask; } diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs index 5f2fb93d..b88730f0 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs @@ -2,6 +2,12 @@ namespace SW.Bitween.Adapters.Bus.Sqs; public class SqsOptions { + /// + /// False creates the client but starts no pollers. This is what a connection test runs as: + /// without it, testing a data source would start receiving from the customer's queue. + /// + public bool Consume { get; set; } = true; + public string Region { get; set; } = "eu-west-1"; /// diff --git a/SW.Bitween.Api/Resources/BusGateways/Create.cs b/SW.Bitween.Api/Resources/BusGateways/Create.cs index fb85fa61..260211c0 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Create.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Create.cs @@ -29,11 +29,16 @@ public async Task Handle(BusGatewayCreate model) if (!documentExists) throw new SWNotFoundException($"Document with Id {model.DocumentId} not found"); + await EnsureDataSourceAsync(_dbContext, model); + var entity = new BusGateway { Name = model.Name, DocumentId = model.DocumentId, - Inactive = model.Inactive + Inactive = model.Inactive, + DataSourceId = model.DataSourceId, + Endpoint = model.DataSourceId == null ? null : model.Endpoint, + EndpointProperties = model.EndpointProperties ?? new() }; _dbContext.Add(entity); @@ -42,11 +47,40 @@ public async Task Handle(BusGatewayCreate model) return entity.Id; } + /// + /// Shared by Create and Update. An endpoint is what the supervisor turns into the adapter's + /// consume list, so an external gateway without one is a gateway that can never receive + /// anything — and it would fail silently, which is the worst way for it to fail. + /// + internal static async Task EnsureDataSourceAsync(BitweenDbContext dbContext, BusGatewayCreate model) + { + if (model.DataSourceId == null) return; + + var exists = await dbContext.Set() + .AnyAsync(d => d.Id == model.DataSourceId); + if (!exists) + throw new SWNotFoundException($"DataSource with Id {model.DataSourceId} not found"); + + if (string.IsNullOrWhiteSpace(model.Endpoint)) + throw new SWException( + "An external bus gateway needs an endpoint — the queue, topic or subscription " + + "on that data source it reads from."); + + // Two gateways consuming one endpoint on one data source would both be offered every + // message, and only the one the sink happens to pick would ever run. + var endpointTaken = await dbContext.Set() + .AnyAsync(g => g.DataSourceId == model.DataSourceId && g.Endpoint == model.Endpoint); + if (endpointTaken) + throw new SWException( + $"Another bus gateway on this data source already reads '{model.Endpoint}'."); + } + private class Validate : AbstractValidator { public Validate() { RuleFor(i => i.Name).NotEmpty().MaximumLength(200); + RuleFor(i => i.Endpoint).MaximumLength(500); } } } diff --git a/SW.Bitween.Api/Resources/BusGateways/Get.cs b/SW.Bitween.Api/Resources/BusGateways/Get.cs index ccaf6aa8..fdb3ac24 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Get.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Get.cs @@ -39,6 +39,14 @@ public async Task Handle(int key) .Select(d => d.Name) .FirstOrDefaultAsync(); + var dataSource = gateway.DataSourceId == null + ? null + : await _dbContext.Set() + .AsNoTracking() + .Where(d => d.Id == gateway.DataSourceId) + .Select(d => new { d.Name, d.LastKnownState }) + .FirstOrDefaultAsync(); + return new BusGatewayRow { Id = gateway.Id, @@ -46,6 +54,11 @@ public async Task Handle(int key) DocumentId = gateway.DocumentId, Inactive = gateway.Inactive, DocumentName = documentName, + DataSourceId = gateway.DataSourceId, + DataSourceName = dataSource?.Name, + DataSourceState = dataSource?.LastKnownState, + Endpoint = gateway.Endpoint, + EndpointProperties = gateway.EndpointProperties ?? new(), RoutesCount = gateway.Routes.Count, Routes = gateway.Routes.Select(r => new BusGatewayRouteDto { diff --git a/SW.Bitween.Api/Resources/BusGateways/Search.cs b/SW.Bitween.Api/Resources/BusGateways/Search.cs index d59daa5b..e3fc37cd 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Search.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Search.cs @@ -28,6 +28,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa await _requestContext.EnsurePermission(_dbContext, Model.Permissions.BusGateways.View); var documents = _dbContext.Set(); + var dataSources = _dbContext.Set(); var query = from gateway in _dbContext.Set() select new BusGatewayRow @@ -38,6 +39,16 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa Inactive = gateway.Inactive, DocumentName = documents.Where(d => d.Id == gateway.DocumentId) .Select(d => d.Name).FirstOrDefault(), + + // Null name means the internal bus, which is what the list column + // reads — an operator should be able to tell at a glance which of + // their gateways reach outside. + DataSourceId = gateway.DataSourceId, + DataSourceName = dataSources.Where(d => d.Id == gateway.DataSourceId) + .Select(d => d.Name).FirstOrDefault(), + DataSourceState = dataSources.Where(d => d.Id == gateway.DataSourceId) + .Select(d => d.LastKnownState).FirstOrDefault(), + Endpoint = gateway.Endpoint, RoutesCount = gateway.Routes.Count }; diff --git a/SW.Bitween.Api/Resources/BusGateways/Update.cs b/SW.Bitween.Api/Resources/BusGateways/Update.cs index 51b0dcaa..ad0a8294 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Update.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Update.cs @@ -30,20 +30,52 @@ public async Task Handle(int key, BusGatewayUpdate model) if (entity == null) throw new SWNotFoundException($"BusGateway with Id {key} not found"); - // Name only; the bound document is fixed at creation (routes' subscriptions belong to it). + await EnsureEndpointFreeAsync(_dbContext, key, model); + + // The bound document is still fixed at creation — routes' subscriptions belong to it — + // but where the messages COME from is exactly the thing an operator needs to change + // without rebuilding the gateway and all of its routes. entity.Name = model.Name; entity.Inactive = model.Inactive; + entity.DataSourceId = model.DataSourceId; + entity.Endpoint = model.DataSourceId == null ? null : model.Endpoint; + entity.EndpointProperties = model.EndpointProperties ?? new(); await _dbContext.SaveChangesAsync(); await _cache.BroadcastRevoke(); return null; } + private static async Task EnsureEndpointFreeAsync( + BitweenDbContext dbContext, int key, BusGatewayUpdate model) + { + if (model.DataSourceId == null) return; + + var exists = await dbContext.Set() + .AnyAsync(d => d.Id == model.DataSourceId); + if (!exists) + throw new SWNotFoundException($"DataSource with Id {model.DataSourceId} not found"); + + if (string.IsNullOrWhiteSpace(model.Endpoint)) + throw new SWException( + "An external bus gateway needs an endpoint — the queue, topic or subscription " + + "on that data source it reads from."); + + var endpointTaken = await dbContext.Set() + .AnyAsync(g => g.DataSourceId == model.DataSourceId + && g.Endpoint == model.Endpoint + && g.Id != key); + if (endpointTaken) + throw new SWException( + $"Another bus gateway on this data source already reads '{model.Endpoint}'."); + } + private class Validate : AbstractValidator { public Validate() { RuleFor(i => i.Name).NotEmpty().MaximumLength(200); + RuleFor(i => i.Endpoint).MaximumLength(500); } } } diff --git a/SW.Bitween.Api/Resources/DataSources/Create.cs b/SW.Bitween.Api/Resources/DataSources/Create.cs new file mode 100644 index 00000000..5120afa6 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Create.cs @@ -0,0 +1,67 @@ +using System; +using System.Threading.Tasks; +using FluentValidation; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.DataSources; + +public class Create : ICommandHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + + public Create(BitweenDbContext dbContext, RequestContext requestContext) + { + _dbContext = dbContext; + _requestContext = requestContext; + } + + public async Task Handle(DataSourceCreate model) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.Create); + + var nameTaken = await _dbContext.Set() + .AnyAsync(d => d.Name == model.Name); + if (nameTaken) + throw new SWException($"A data source named '{model.Name}' already exists."); + + // Nothing is stored behind a sentinel on a create, so any that arrives — from a data source + // copied out of Get, say — is dropped rather than saved as the literal string. + var properties = Secrets.Merge(null, model.Properties); + + var entity = new DataSource + { + Name = model.Name, + AdapterId = model.AdapterId, + Kind = ParseKind(model.Kind), + Properties = properties, + SecretProperties = Secrets.Declare(properties, model.SecretProperties), + Inactive = model.Inactive, + DeduplicationWindowDays = model.DeduplicationWindowDays + }; + + _dbContext.Add(entity); + await _dbContext.SaveChangesAsync(); + return entity.Id; + } + + internal static DataSourceKind ParseKind(string kind) => + Enum.TryParse(kind, ignoreCase: true, out var parsed) + ? parsed + : DataSourceKind.Broker; + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(i => i.Name).NotEmpty().MaximumLength(200); + RuleFor(i => i.AdapterId).NotEmpty().MaximumLength(200); + + // Zero is meaningful — it turns deduplication off — so only a negative is rejected. + RuleFor(i => i.DeduplicationWindowDays).GreaterThanOrEqualTo(0); + } + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Delete.cs b/SW.Bitween.Api/Resources/DataSources/Delete.cs new file mode 100644 index 00000000..dadb29ec --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Delete.cs @@ -0,0 +1,46 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.EfCoreExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.DataSources; + +public class Delete : IDeleteHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + + public Delete(BitweenDbContext dbContext, RequestContext requestContext) + { + _dbContext = dbContext; + _requestContext = requestContext; + } + + public async Task Handle(int key) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.Delete); + + // The database refuses this anyway — the foreign key restricts — but a raw constraint + // violation tells an operator nothing about which gateway is in the way. + var gateways = await _dbContext.Set() + .Where(gateway => gateway.DataSourceId == key) + .Select(gateway => gateway.Name) + .Take(5) + .ToListAsync(); + + if (gateways.Count > 0) + throw new SWException( + "Cannot delete a data source that still feeds bus gateways: " + + string.Join(", ", gateways) + + ". Point them at the internal bus, or delete them, first."); + + // Dedupe keys cascade with the data source, which is what makes deleting and recreating a + // data source a genuine reset rather than one that silently suppresses the first messages. + await _dbContext.DeleteByKeyAsync(key); + await _dbContext.SaveChangesAsync(); + return null; + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Get.cs b/SW.Bitween.Api/Resources/DataSources/Get.cs new file mode 100644 index 00000000..1709b14e --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Get.cs @@ -0,0 +1,57 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.DataSources; + +public class Get : IGetHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + + public Get(BitweenDbContext dbContext, RequestContext requestContext) + { + _dbContext = dbContext; + _requestContext = requestContext; + } + + public async Task Handle(int key) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.View); + + var dataSource = await _dbContext.Set() + .AsNoTracking() + .FirstOrDefaultAsync(d => d.Id == key); + + if (dataSource == null) + throw new SWNotFoundException($"DataSource with id '{key}' was not found"); + + return new DataSourceRow + { + Id = dataSource.Id, + Name = dataSource.Name, + AdapterId = dataSource.AdapterId, + Kind = dataSource.Kind.ToString(), + Inactive = dataSource.Inactive, + DeduplicationWindowDays = dataSource.DeduplicationWindowDays, + + // Masked, always. This is the only endpoint that returns connection settings, so it is + // the only place a broker password could leave the process. + Properties = Secrets.Mask(dataSource.Properties, dataSource.SecretProperties), + SecretProperties = dataSource.SecretProperties, + + LastKnownState = dataSource.LastKnownState, + LastHeartbeatOn = dataSource.LastHeartbeatOn, + LastException = dataSource.LastException, + ConsecutiveFailures = dataSource.ConsecutiveFailures, + OwnedByNode = dataSource.OwnedByNode, + + GatewayCount = await _dbContext.Set() + .CountAsync(gateway => gateway.DataSourceId == key) + }; + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Search.cs b/SW.Bitween.Api/Resources/DataSources/Search.cs new file mode 100644 index 00000000..2f4323b6 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Search.cs @@ -0,0 +1,67 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.Model; +using SW.EfCoreExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.DataSources; + +public class Search : ISearchyHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + + public Search(BitweenDbContext dbContext, RequestContext requestContext) + { + _dbContext = dbContext; + _requestContext = requestContext; + } + + public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) + { + // Lookup is id/name pairs, which the bus gateway picker needs in order to offer a data + // source at all; the full list carries connection detail, so that is what View covers. + if (!lookup) + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.View); + + var query = from dataSource in _dbContext.Set() + select new DataSourceRow + { + Id = dataSource.Id, + Name = dataSource.Name, + AdapterId = dataSource.AdapterId, + Kind = dataSource.Kind.ToString(), + Inactive = dataSource.Inactive, + DeduplicationWindowDays = dataSource.DeduplicationWindowDays, + LastKnownState = dataSource.LastKnownState, + LastHeartbeatOn = dataSource.LastHeartbeatOn, + LastException = dataSource.LastException, + ConsecutiveFailures = dataSource.ConsecutiveFailures, + OwnedByNode = dataSource.OwnedByNode, + + // A correlated count, so the "used by" column costs one subquery per row rather + // than the whole BusGateway table over the wire. + GatewayCount = _dbContext.Set() + .Count(gateway => gateway.DataSourceId == dataSource.Id) + }; + + query = query.AsNoTracking(); + + // Properties are deliberately absent from the list: it is a table of connections, and + // sending every credential — even masked — to render a row is not worth the exposure. + + if (lookup) + return await query.Search(searchyRequest.Conditions) + .ToDictionaryAsync(k => k.Id.ToString(), v => v.Name); + + return new SearchyResponse + { + TotalCount = await query.Search(searchyRequest.Conditions).CountAsync(), + Result = await query.Search(searchyRequest.Conditions, searchyRequest.Sorts, + searchyRequest.PageSize, searchyRequest.PageIndex).ToListAsync() + }; + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Secrets.cs b/SW.Bitween.Api/Resources/DataSources/Secrets.cs new file mode 100644 index 00000000..c7208fa1 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Secrets.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SW.Bitween.Resources.DataSources; + +/// +/// Keeps a data source's credentials out of responses. +/// +/// This does NOT ask the adapter which of its startup values are marked secure, the way +/// does for subscription properties. Describing an adapter +/// means starting it, and a bus provider is resident — started once and held for the life of the +/// node — so asking would either start a second copy of a connection that is meant to be exclusive +/// or block on one that is already running. The data source names its own secrets instead, which +/// also lets an operator protect a field the adapter author never thought to mark. +/// +public static class Secrets +{ + /// Stands in for a stored secret. Deliberately the same sentinel the rest of the app uses. + public const string Sentinel = AdapterSecretProperties.Sentinel; + + /// + /// Names that are treated as secret whether or not anyone listed them. A credential missed + /// because nobody ticked a box is a credential in a JSON response, so the default is to hide. + /// + private static readonly string[] AlwaysSecret = + [ + "password", "secret", "token", "credential", "apikey", "accesskey", + "privatekey", "connectionstring", "sas", "passphrase", "certificate" + ]; + + public static bool IsSecret(string name, IEnumerable declared) => + (declared ?? []).Contains(name, StringComparer.OrdinalIgnoreCase) || + AlwaysSecret.Any(fragment => name.Contains(fragment, StringComparison.OrdinalIgnoreCase)); + + /// + /// A copy with every secret replaced. An empty value is left alone, so "not set" stays + /// distinguishable from "set but hidden" — the difference between a form that is finished and + /// one that is not. + /// + public static Dictionary Mask( + IReadOnlyDictionary properties, IEnumerable declared) + { + if (properties == null) return new Dictionary(); + + var secretNames = declared?.ToList() ?? []; + return properties.ToDictionary(kv => kv.Key, + kv => IsSecret(kv.Key, secretNames) && !string.IsNullOrEmpty(kv.Value) + ? Sentinel + : kv.Value); + } + + /// + /// Resolves sentinels against what is stored. A sentinel with nothing behind it is dropped + /// rather than saved literally — otherwise a data source copied from a response would + /// authenticate with the string "__private__". + /// + public static Dictionary Merge( + IReadOnlyDictionary stored, IReadOnlyDictionary incoming) + { + var result = new Dictionary(); + if (incoming == null) return result; + + foreach (var kv in incoming) + { + if (kv.Value != Sentinel) result[kv.Key] = kv.Value; + else if (stored != null && stored.TryGetValue(kv.Key, out var storedValue)) + result[kv.Key] = storedValue; + } + + return result; + } + + /// + /// The secret names to record. Whatever the caller declared, plus anything matching a + /// well-known credential name — so the list stays true even when the form did not tick it. + /// + public static List Declare( + IReadOnlyDictionary properties, IEnumerable declared) + { + var names = new HashSet(declared ?? [], StringComparer.OrdinalIgnoreCase); + + foreach (var key in properties?.Keys ?? Enumerable.Empty()) + if (AlwaysSecret.Any(f => key.Contains(f, StringComparison.OrdinalIgnoreCase))) + names.Add(key); + + return names.ToList(); + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Telemetry.cs b/SW.Bitween.Api/Resources/DataSources/Telemetry.cs new file mode 100644 index 00000000..48797885 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Telemetry.cs @@ -0,0 +1,92 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; + +namespace SW.Bitween.Resources.DataSources; + +/// +/// What this connection is doing right now. +/// +/// The data source row carries a summary written back on the reconcile loop, which is up to thirty +/// seconds old and deliberately small. This reads the heartbeat directly, so an operator watching a +/// queue drain sees it drain — and gets the two things the row has never carried: the per-queue +/// depths the adapter can see, and the host-observed process figures that keep working when the +/// adapter is wedged and reporting nothing at all. +/// +/// Scoped to THIS node, and says so. A broker connection is exclusive, so at most one node runs any +/// given adapter; every other node answers RunningHere=false rather than inventing an outage. +/// +[HandlerName("telemetry")] +public class Telemetry : IGetHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + private readonly IResidentAdapterHost _adapters; + + public Telemetry(BitweenDbContext dbContext, RequestContext requestContext, + IResidentAdapterHost adapters = null) + { + _dbContext = dbContext; + _requestContext = requestContext; + _adapters = adapters; + } + + public async Task Handle(int key) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.View); + + var dataSource = await _dbContext.Set().AsNoTracking() + .FirstOrDefaultAsync(d => d.Id == key); + + if (dataSource == null) + throw new SWNotFoundException($"DataSource with id '{key}' was not found"); + + var telemetry = new DataSourceTelemetry + { + OwnedByNode = dataSource.OwnedByNode, + + // The row's own summary, so the answer is still useful when the adapter runs elsewhere + // or has not started. Overwritten below by the live heartbeat when there is one. + State = dataSource.LastKnownState, + LastError = dataSource.LastException, + RestartCount = dataSource.ConsecutiveFailures, + LastHeartbeatOn = dataSource.LastHeartbeatOn, + }; + + // Registered only when BusProvidersEnabled, so a node with the feature off answers + // "not here" rather than failing to resolve a service. + var instance = _adapters?.Describe() + .FirstOrDefault(h => h.InstanceKey == key.ToString()); + + if (instance == null) return telemetry; + + telemetry.RunningHere = true; + telemetry.Connected = instance.Connected; + telemetry.State = instance.ReportedState ?? instance.State.ToString(); + telemetry.LastMessageOn = instance.LastMessageOn?.UtcDateTime; + telemetry.InFlight = instance.InFlight; + telemetry.LastError = instance.LastError ?? dataSource.LastException; + + telemetry.ProcessId = instance.ProcessId; + telemetry.WorkingSetBytes = instance.WorkingSetBytes; + telemetry.CpuPercent = instance.CpuPercent; + telemetry.ThreadCount = instance.ThreadCount; + telemetry.Uptime = instance.Uptime; + telemetry.RestartCount = instance.RestartCount; + telemetry.MissedHeartbeats = instance.MissedHeartbeats; + telemetry.Quarantined = instance.Quarantined; + telemetry.LastHeartbeatOn = instance.LastHeartbeatOn?.UtcDateTime ?? dataSource.LastHeartbeatOn; + + foreach (var kv in instance.Details ?? new System.Collections.Generic.Dictionary()) + telemetry.Details[kv.Key] = kv.Value; + + telemetry.Commands = (instance.Commands ?? Array.Empty()).ToList(); + + return telemetry; + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Test.cs b/SW.Bitween.Api/Resources/DataSources/Test.cs new file mode 100644 index 00000000..76d80c3a --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Test.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; + +namespace SW.Bitween.Resources.DataSources; + +/// +/// Reaches out to the broker and reports what happened, stage by stage. +/// +/// This is the one control that turns configuring a data source from guesswork into something an +/// operator can finish: the alternative is to save it, wait up to thirty seconds for the supervisor +/// to reconcile, and then read the health column to learn that the password was wrong. +/// +/// It runs the real adapter against the real settings, because a check that reimplements the +/// broker's handshake only proves that the reimplementation agrees with itself. What it does NOT do +/// is consume: the instance is started with Consume=false, so the customer's queue is inspected and +/// never drained, and it is stopped again before this returns. +/// +[HandlerName("test")] +public class Test : ICommandHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + private readonly IResidentAdapterHost _adapters; + + public Test(BitweenDbContext dbContext, RequestContext requestContext, + IResidentAdapterHost adapters = null) + { + _dbContext = dbContext; + _requestContext = requestContext; + _adapters = adapters; + } + + public async Task Handle(int key, DataSourceTestRequest request) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.Operate); + + // Registered only when BusProvidersEnabled, so say which switch is off rather than + // failing to resolve a service the operator has never heard of. + if (_adapters == null) + return Failed("External bus providers are turned off on this node " + + "(Bitween:BusProvidersEnabled). Nothing can connect from here."); + + var dataSource = await _dbContext.Set().AsNoTracking() + .FirstOrDefaultAsync(d => d.Id == key); + + if (dataSource == null) + throw new SWNotFoundException($"DataSource with id '{key}' was not found"); + + var startupValues = new Dictionary( + dataSource.Properties ?? new Dictionary(), + StringComparer.OrdinalIgnoreCase); + + // The endpoints its gateways want, so the test checks the queues that will actually be + // used rather than only that the credentials work. + var endpoints = await _dbContext.Set() + .Where(g => g.DataSourceId == key && !g.Inactive && g.Endpoint != null) + .Select(g => g.Endpoint) + .Distinct() + .ToListAsync(); + + if (endpoints.Count > 0) startupValues["Endpoints"] = string.Join(",", endpoints); + startupValues["Consume"] = "false"; + + // A key of its own: the running instance, if there is one, is keyed by the data source id + // and must not be disturbed by someone pressing Test. + var instanceKey = $"test-{key}-{Guid.NewGuid():N}"[..24]; + + try + { + var instance = await _adapters.StartExclusiveAsync(new AdapterSpec + { + AdapterId = dataSource.AdapterId, + InstanceKey = instanceKey, + StartupValues = startupValues + }); + + var raw = await instance.InvokeAsync("TestConnection", timeoutSeconds: 30); + return Translate(raw); + } + catch (Exception ex) + { + // A broker that cannot be reached throws on start, before any stage runs. That is a + // result, not a server error: the operator asked whether it works, and it does not. + return Failed(ex.Message); + } + finally + { + try { await _adapters.StopAsync(dataSource.AdapterId, instanceKey, drain: false); } + catch { /* the instance may never have started */ } + } + } + + /// + /// The adapters answer with { ok, steps: [{ step, ok, detail }] }, which is their shape rather + /// than Bitween's. Translating here keeps the API contract stable while adapters evolve. + /// + private static DataSourceTestResult Translate(JObject raw) + { + if (raw == null) return Failed("The adapter returned nothing."); + + var result = new DataSourceTestResult { Succeeded = raw.Value("ok") }; + + foreach (var step in raw["steps"] as JArray ?? []) + result.Stages.Add(new DataSourceTestStage + { + Name = step.Value("step"), + Succeeded = step.Value("ok"), + Detail = step.Value("detail") + }); + + if (!result.Succeeded) + result.Error = result.Stages.LastOrDefault(s => !s.Succeeded)?.Detail + ?? "The adapter reported a failure without naming a stage."; + + return result; + } + + private static DataSourceTestResult Failed(string error) => + new() { Succeeded = false, Error = error }; +} diff --git a/SW.Bitween.Api/Resources/DataSources/Update.cs b/SW.Bitween.Api/Resources/DataSources/Update.cs new file mode 100644 index 00000000..d157a490 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Update.cs @@ -0,0 +1,62 @@ +using System.Threading.Tasks; +using FluentValidation; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.DataSources; + +public class Update : ICommandHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + + public Update(BitweenDbContext dbContext, RequestContext requestContext) + { + _dbContext = dbContext; + _requestContext = requestContext; + } + + public async Task Handle(int key, DataSourceUpdate model) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.Edit); + + var entity = await _dbContext.Set().FirstOrDefaultAsync(d => d.Id == key); + if (entity == null) + throw new SWNotFoundException($"DataSource with Id {key} not found"); + + var nameTaken = await _dbContext.Set() + .AnyAsync(d => d.Name == model.Name && d.Id != key); + if (nameTaken) + throw new SWException($"A data source named '{model.Name}' already exists."); + + // Secrets came out of Get masked, so put them back from what is stored. A form that only + // changed the prefetch must not overwrite the password with a row of dots. + var properties = Secrets.Merge(entity.Properties, model.Properties); + + entity.Name = model.Name; + entity.AdapterId = model.AdapterId; + entity.Kind = Create.ParseKind(model.Kind); + entity.Properties = properties; + entity.SecretProperties = Secrets.Declare(properties, model.SecretProperties); + entity.Inactive = model.Inactive; + entity.DeduplicationWindowDays = model.DeduplicationWindowDays; + + await _dbContext.SaveChangesAsync(); + + // No cache to revoke and no adapter to restart from here: the supervisor reconciles against + // these rows on its own loop, notices the fingerprint changed, and restarts the adapter. + return null; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(i => i.Name).NotEmpty().MaximumLength(200); + RuleFor(i => i.AdapterId).NotEmpty().MaximumLength(200); + RuleFor(i => i.DeduplicationWindowDays).GreaterThanOrEqualTo(0); + } + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs b/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs new file mode 100644 index 00000000..0f2252dc --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs @@ -0,0 +1,432 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.Domain; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The API an operator configures an external bus gateway through. +/// +/// Until this existed the whole external-bus feature was reachable only by writing rows into the +/// database by hand, which is how every other test in this suite still sets itself up. The +/// questions worth asking of it are about credentials — a broker password must not come back out +/// of an endpoint, and it must survive an edit that never touched it — and about the two ways a +/// gateway can be wrong: pointed at nothing, or pointed at a queue another gateway already reads. +/// +[Collection("Bitween")] +public class DataSourceApiTests +{ + private readonly BitweenFixture _fixture; + + public DataSourceApiTests(BitweenFixture fixture) => _fixture = fixture; + + // ---------------------------------------------------------------- secrets + + /// + /// A password goes in and does not come back. This is the entire reason Get masks: the data + /// source screen is the only place connection settings are ever served, so it is the only + /// place a customer's broker credentials could leave the process. + /// + [Fact] + public async Task A_secret_property_is_never_returned_in_clear() + { + var id = await CreateAsync(new Dictionary + { + ["Host"] = "broker.example.com", + ["UserName"] = "bitween", + ["Password"] = "hunter2" + }); + + var row = await GetAsync(id); + + Assert.Equal("broker.example.com", row.Properties["Host"]); + Assert.Equal("bitween", row.Properties["UserName"]); + Assert.Equal(AdapterSecretProperties.Sentinel, row.Properties["Password"]); + Assert.DoesNotContain("hunter2", string.Join("|", row.Properties.Values)); + } + + /// + /// Nobody ticked a box; the property is still a password. Relying on an operator to declare + /// every credential means the one they forget is the one in the JSON response. + /// + [Fact] + public async Task A_credential_is_masked_even_when_nobody_declared_it() + { + var id = await CreateAsync(new Dictionary + { + ["Region"] = "eu-west-1", + ["SecretAccessKey"] = "abc/123", + ["AccessKeyId"] = "AKIAEXAMPLE" + }, secretProperties: []); // declared nothing + + var row = await GetAsync(id); + + Assert.Equal("eu-west-1", row.Properties["Region"]); + Assert.Equal(AdapterSecretProperties.Sentinel, row.Properties["SecretAccessKey"]); + Assert.Equal(AdapterSecretProperties.Sentinel, row.Properties["AccessKeyId"]); + + // And it is recorded as secret, so the next reader does not have to rediscover it. + Assert.Contains("SecretAccessKey", row.SecretProperties); + } + + /// + /// The round trip that breaks naive masking: read, change one unrelated field, save. If the + /// sentinel were stored literally the broker would start authenticating with "__private__" and + /// the only symptom would be an integration that stopped working. + /// + [Fact] + public async Task Saving_a_masked_secret_back_keeps_the_stored_value() + { + var id = await CreateAsync(new Dictionary + { + ["Host"] = "broker.example.com", + ["Password"] = "hunter2" + }); + + var row = await GetAsync(id); + row.Properties["Host"] = "broker2.example.com"; // the only real edit + + await UpdateAsync(id, row); + + Assert.Equal("hunter2", await StoredPropertyAsync(id, "Password")); + Assert.Equal("broker2.example.com", await StoredPropertyAsync(id, "Host")); + } + + /// And a genuine change still goes through, or the field would be uneditable. + [Fact] + public async Task A_secret_that_was_actually_changed_is_saved() + { + var id = await CreateAsync(new Dictionary { ["Password"] = "old" }); + + var row = await GetAsync(id); + row.Properties["Password"] = "new-one"; + await UpdateAsync(id, row); + + Assert.Equal("new-one", await StoredPropertyAsync(id, "Password")); + } + + /// + /// A data source pasted from a Get response — which is how anyone scripts a second environment + /// — must not authenticate with the sentinel. There is nothing stored to restore from, so it + /// is dropped and the field reads as unset rather than as a password nobody chose. + /// + [Fact] + public async Task A_sentinel_with_nothing_behind_it_is_dropped_rather_than_stored() + { + var id = await CreateAsync(new Dictionary + { + ["Host"] = "broker.example.com", + ["Password"] = AdapterSecretProperties.Sentinel + }); + + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var stored = await db.Set().AsNoTracking().FirstAsync(d => d.Id == id); + + Assert.False(stored.Properties.ContainsKey("Password"), + "the sentinel was stored as if it were the password"); + } + + /// + /// The list is a table of connections, not of credentials. Even masked, sending every + /// property to render a row is exposure with no purpose. + /// + [Fact] + public async Task The_list_carries_no_connection_properties_at_all() + { + var id = await CreateAsync(new Dictionary { ["Password"] = "hunter2" }); + + var rows = await SearchAsync(); + var row = rows.Single(r => r.Id == id); + + Assert.True(row.Properties == null || row.Properties.Count == 0); + Assert.Equal(BusAdapters.RabbitMq, row.AdapterId); + } + + // ---------------------------------------------------------------- lifecycle + + [Fact] + public async Task A_data_source_name_is_unique() + { + var name = Unique("ds"); + await CreateAsync(new Dictionary(), name: name); + + var error = await Assert.ThrowsAnyAsync( + () => CreateAsync(new Dictionary(), name: name)); + + Assert.Contains("already exists", error.Message); + } + + /// + /// The database refuses this too, but a raw foreign-key violation names a constraint rather + /// than the gateway standing in the way, and an operator cannot act on a constraint name. + /// + [Fact] + public async Task Deleting_a_data_source_that_feeds_a_gateway_says_which_gateway() + { + var id = await CreateAsync(new Dictionary()); + var gatewayName = Unique("gw"); + await CreateGatewayAsync(id, Unique("q"), gatewayName); + + var error = await Assert.ThrowsAnyAsync(() => DeleteAsync(id)); + + Assert.Contains(gatewayName, error.Message); + } + + [Fact] + public async Task A_data_source_nothing_uses_can_be_deleted() + { + var id = await CreateAsync(new Dictionary()); + await DeleteAsync(id); + + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + Assert.False(await db.Set().AnyAsync(d => d.Id == id)); + } + + // ---------------------------------------------------------------- gateways + + /// + /// The default that keeps every gateway already in a database working: no data source means + /// the internal bus, exactly as before. + /// + [Fact] + public async Task A_gateway_created_without_a_data_source_is_internal() + { + var documentId = await CreateDocumentAsync(); + + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var create = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var gatewayId = (int)await create.Handle(new BusGatewayCreate + { + Name = Unique("gw"), DocumentId = documentId + }); + + var row = await GetGatewayAsync(gatewayId); + + Assert.Null(row.DataSourceId); + Assert.Null(row.DataSourceName); + Assert.Null(row.Endpoint); + } + + [Fact] + public async Task A_gateway_pointed_at_a_data_source_reports_it() + { + var dataSourceId = await CreateAsync(new Dictionary()); + var queue = Unique("q"); + var gatewayId = await CreateGatewayAsync(dataSourceId, queue); + + var row = await GetGatewayAsync(gatewayId); + + Assert.Equal(dataSourceId, row.DataSourceId); + Assert.Equal(queue, row.Endpoint); + Assert.NotNull(row.DataSourceName); + } + + /// + /// An external gateway with no endpoint is a gateway that can never receive anything: the + /// supervisor builds the adapter's consume list from endpoints, so it would sit there + /// connected and idle, with nothing anywhere saying why. + /// + [Fact] + public async Task An_external_gateway_without_an_endpoint_is_refused() + { + var dataSourceId = await CreateAsync(new Dictionary()); + var documentId = await CreateDocumentAsync(); + + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var create = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + var error = await Assert.ThrowsAnyAsync(() => create.Handle(new BusGatewayCreate + { + Name = Unique("gw"), DocumentId = documentId, DataSourceId = dataSourceId + })); + + Assert.Contains("endpoint", error.Message, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Two gateways on one endpoint would both be candidates for every message and only one would + /// ever run — a silent misroute rather than an error, which is the failure mode this whole + /// area keeps producing when it is left unguarded. + /// + [Fact] + public async Task Two_gateways_cannot_read_the_same_endpoint_on_one_data_source() + { + var dataSourceId = await CreateAsync(new Dictionary()); + var queue = Unique("q"); + await CreateGatewayAsync(dataSourceId, queue); + + var error = await Assert.ThrowsAnyAsync( + () => CreateGatewayAsync(dataSourceId, queue)); + + Assert.Contains("already reads", error.Message); + } + + /// Moving a gateway between the internal bus and a broker is the point of the field. + [Fact] + public async Task A_gateway_can_be_moved_from_internal_to_external_and_back() + { + var dataSourceId = await CreateAsync(new Dictionary()); + var documentId = await CreateDocumentAsync(); + var queue = Unique("q"); + + int gatewayId; + await using (var scope = _fixture.CreateScope()) + { + scope.Superuser(); + var create = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + gatewayId = (int)await create.Handle(new BusGatewayCreate + { + Name = Unique("gw"), DocumentId = documentId + }); + } + + await UpdateGatewayAsync(gatewayId, dataSourceId, queue); + var external = await GetGatewayAsync(gatewayId); + Assert.Equal(dataSourceId, external.DataSourceId); + Assert.Equal(queue, external.Endpoint); + + await UpdateGatewayAsync(gatewayId, dataSourceId: null, endpoint: null); + var internalAgain = await GetGatewayAsync(gatewayId); + Assert.Null(internalAgain.DataSourceId); + + // The endpoint goes with it. Leaving one behind would show an internal gateway claiming to + // read a queue, which is the sort of thing that survives for a year before anyone asks. + Assert.Null(internalAgain.Endpoint); + } + + // ---------------------------------------------------------------- helpers + + private static string Unique(string prefix) => $"{prefix}-{Guid.NewGuid():N}"[..20]; + + private async Task CreateAsync(Dictionary properties, + string name = null, List secretProperties = null) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + return (int)await handler.Handle(new DataSourceCreate + { + Name = name ?? Unique("ds"), + AdapterId = BusAdapters.RabbitMq, + Kind = "Broker", + Properties = properties, + SecretProperties = secretProperties ?? ["Password"] + }); + } + + private async Task GetAsync(int id) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + return (DataSourceRow)await handler.Handle(id); + } + + private async Task UpdateAsync(int id, DataSourceRow row) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + await handler.Handle(id, new DataSourceUpdate + { + Name = row.Name, + AdapterId = row.AdapterId, + Kind = row.Kind, + Properties = row.Properties, + SecretProperties = row.SecretProperties, + Inactive = row.Inactive, + DeduplicationWindowDays = row.DeduplicationWindowDays + }); + } + + private async Task DeleteAsync(int id) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + await handler.Handle(id); + } + + private async Task> SearchAsync() + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var response = (SearchyResponse)await handler.Handle(new SearchyRequest { PageSize = 500 }); + return response.Result.ToList(); + } + + private async Task StoredPropertyAsync(int id, string name) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var stored = await db.Set().AsNoTracking().FirstAsync(d => d.Id == id); + return stored.Properties.TryGetValue(name, out var value) ? value : null; + } + + private async Task CreateDocumentAsync() + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var document = new Document(null, Unique("doc"), DocumentFormat.Json); + db.Add(document); + await db.SaveChangesAsync(); + return document.Id; + } + + private async Task CreateGatewayAsync(int dataSourceId, string endpoint, string name = null) + { + var documentId = await CreateDocumentAsync(); + + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + return (int)await handler.Handle(new BusGatewayCreate + { + Name = name ?? Unique("gw"), + DocumentId = documentId, + DataSourceId = dataSourceId, + Endpoint = endpoint + }); + } + + private async Task UpdateGatewayAsync(int gatewayId, int? dataSourceId, string endpoint) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + var current = await GetGatewayAsync(gatewayId); + await handler.Handle(gatewayId, new BusGatewayUpdate + { + Name = current.Name, + DocumentId = current.DocumentId, + DataSourceId = dataSourceId, + Endpoint = endpoint + }); + } + + private async Task GetGatewayAsync(int gatewayId) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + return (BusGatewayRow)await handler.Handle(gatewayId); + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/SharedBrokerTests.cs b/SW.Bitween.IntegrationTests/Tests/SharedBrokerTests.cs new file mode 100644 index 00000000..fec416a8 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/SharedBrokerTests.cs @@ -0,0 +1,630 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using SW.Bitween.Domain; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.Bitween.Services.Cluster; +using SW.Bitween.Services.DataSources; +using Newtonsoft.Json; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// One customer broker, several integrations — which is what a real one looks like. +/// +/// Everything else in this suite exercises a data source with a single queue behind it. That is +/// the easy case and it hides the questions that actually matter once a second integration lands +/// on the same broker: does each queue reach the right information type, does one connection serve +/// all of them or does each gateway open its own, does a queue Bitween was never pointed at stay +/// untouched, and can a gateway be added or taken away without disturbing the ones beside it. +/// +/// Acme's broker here has five queues. Bitween is pointed at three of them, running three +/// different integrations; the other two belong to something else entirely and Bitween must behave +/// as if they are not there. +/// +[Collection("Bitween")] +public class SharedBrokerTests +{ + private const string EchoHandler = "sw.bitween.samplehandler"; + + private readonly BitweenFixture _fixture; + + public SharedBrokerTests(BitweenFixture fixture) => _fixture = fixture; + + /// + /// The multiplexing claim, stated as a number: three gateways, one process. + /// + /// If each gateway opened its own connection, a customer with twenty integrations on one broker + /// would have Bitween holding twenty connections and twenty adapter processes — and every one + /// of them would need its own lease, its own restart budget and its own credentials in memory. + /// The whole point of separating the connection from what is done with it is that this stays + /// one of each. + /// + [Fact] + public async Task One_connection_and_one_process_serve_every_gateway_on_the_broker() + { + var acme = await BrokerAsync("acme"); + var orders = await IntegrationAsync(acme, "orders"); + var invoices = await IntegrationAsync(acme, "invoices"); + var shipments = await IntegrationAsync(acme, "shipments"); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var instances = Instances(acme); + Assert.Single(instances); + Assert.NotNull(instances[0].ProcessId); + + Publish(orders.Queue, """{"kind":"order"}"""); + Publish(invoices.Queue, """{"kind":"invoice"}"""); + Publish(shipments.Queue, """{"kind":"shipment"}"""); + + await WaitAsync(async () => + await CountAsync(orders.DocumentId) == 1 && + await CountAsync(invoices.DocumentId) == 1 && + await CountAsync(shipments.DocumentId) == 1, + TimeSpan.FromSeconds(60), "not every queue produced an Xchange"); + + // Still one process after all three have been served. + Assert.Single(Instances(acme)); + } + + /// + /// Three queues, three information types, and no crossing over. + /// + /// This is the failure that would be invisible: a message filed against the wrong information + /// type runs the wrong subscriptions, against a mapping written for a different shape, and + /// nothing anywhere reports an error. Ten messages per queue, interleaved, so the answer does + /// not depend on them arriving one at a time. + /// + [Fact] + public async Task Every_queue_lands_on_its_own_information_type() + { + const int each = 10; + + var acme = await BrokerAsync("route"); + var orders = await IntegrationAsync(acme, "orders"); + var invoices = await IntegrationAsync(acme, "invoices"); + var shipments = await IntegrationAsync(acme, "shipments"); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + // Interleaved rather than queue by queue: three consumers on one channel handling + // deliveries at the same time is the state that finds a routing bug. + using (var connection = ExternalConnection()) + using (var channel = connection.CreateModel()) + { + for (var i = 0; i < each; i++) + foreach (var (queue, kind) in new[] + { + (orders.Queue, "order"), (invoices.Queue, "invoice"), + (shipments.Queue, "shipment"), + }) + PublishOn(channel, queue, $$"""{"kind":"{{kind}}","n":{{i}}}"""); + } + + await WaitAsync(async () => + await CountAsync(orders.DocumentId) >= each && + await CountAsync(invoices.DocumentId) >= each && + await CountAsync(shipments.DocumentId) >= each, + TimeSpan.FromSeconds(90), "the queues did not all drain"); + + // Settle, then assert nothing extra landed anywhere — a crossed-over message shows up as a + // surplus on one document and a shortfall on another. + await Task.Delay(TimeSpan.FromSeconds(3)); + + Assert.Equal(each, await CountAsync(orders.DocumentId)); + Assert.Equal(each, await CountAsync(invoices.DocumentId)); + Assert.Equal(each, await CountAsync(shipments.DocumentId)); + + // And every payload is the one that belongs to its queue. + foreach (var (documentId, kind) in new[] + { + (orders.DocumentId, "order"), (invoices.DocumentId, "invoice"), + (shipments.DocumentId, "shipment"), + }) + Assert.All(await ReferencesAsync(documentId), r => Assert.NotNull(r)); + } + + /// + /// The customer's broker is not Bitween's broker. + /// + /// Acme runs other things on it — a queue their warehouse app drains, another their own + /// services talk over. Bitween is pointed at three queues and must touch nothing else: an + /// adapter that consumed everything it could see would silently eat another system's traffic, + /// and the first anyone would know is when the warehouse stopped receiving orders. + /// + [Fact] + public async Task A_queue_nobody_pointed_Bitween_at_is_left_alone() + { + var acme = await BrokerAsync("bystander"); + var orders = await IntegrationAsync(acme, "orders"); + + // Acme's own traffic, on the same broker, with messages already waiting. + var theirs = Unique("acme-warehouse"); + DeclareQueue(theirs); + for (var i = 0; i < 5; i++) Publish(theirs, $$"""{"theirs":{{i}}}"""); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + Publish(orders.Queue, """{"kind":"order"}"""); + await WaitAsync(async () => await CountAsync(orders.DocumentId) == 1, + TimeSpan.FromSeconds(60), "Bitween's own queue never produced an Xchange"); + + // Bitween has demonstrably been running against this broker, so an untouched depth here + // means it left the queue alone rather than that nothing happened at all. + await Task.Delay(TimeSpan.FromSeconds(3)); + Assert.Equal(5u, Depth(theirs)); + } + + /// + /// A fourth integration goes live on Monday. The three already running must not notice. + /// + /// The adapter restarts to pick up the new consume list — that is expected, and safe, because + /// nothing is acknowledged until Bitween has persisted it. What must NOT happen is the other + /// three losing messages or needing to be reconfigured. + /// + [Fact] + public async Task A_new_integration_joins_the_same_connection_without_disturbing_the_others() + { + var acme = await BrokerAsync("joiner"); + var orders = await IntegrationAsync(acme, "orders"); + var invoices = await IntegrationAsync(acme, "invoices"); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + Publish(orders.Queue, """{"before":true}"""); + await WaitAsync(async () => await CountAsync(orders.DocumentId) == 1, + TimeSpan.FromSeconds(60), "the first integration never worked"); + + // Monday. + var returns = await IntegrationAsync(acme, "returns"); + await supervisor.ReconcileAsync(); + + // Still one connection: the new gateway joined it rather than opening a second. + Assert.Single(Instances(acme)); + + Publish(returns.Queue, """{"kind":"return"}"""); + Publish(orders.Queue, """{"after":true}"""); + Publish(invoices.Queue, """{"kind":"invoice"}"""); + + await WaitAsync(async () => + await CountAsync(returns.DocumentId) == 1 && + await CountAsync(orders.DocumentId) == 2 && + await CountAsync(invoices.DocumentId) == 1, + TimeSpan.FromSeconds(90), "the existing integrations stopped working when a new one joined"); + } + + /// + /// Turning one integration off leaves the rest running. + /// + /// Deactivating a gateway has to stop its queue being consumed — otherwise Bitween keeps + /// acknowledging messages that no longer route anywhere, which is a silent drop rather than a + /// pause — while every other queue on the same connection carries on. + /// + [Fact] + public async Task Deactivating_one_gateway_stops_only_its_queue() + { + var acme = await BrokerAsync("pause"); + var orders = await IntegrationAsync(acme, "orders"); + var invoices = await IntegrationAsync(acme, "invoices"); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + Publish(orders.Queue, """{"n":1}"""); + Publish(invoices.Queue, """{"n":1}"""); + await WaitAsync(async () => + await CountAsync(orders.DocumentId) == 1 && await CountAsync(invoices.DocumentId) == 1, + TimeSpan.FromSeconds(60), "both integrations did not start working"); + + await SetGatewayInactiveAsync(invoices.GatewayId, true); + await supervisor.ReconcileAsync(); + + Publish(orders.Queue, """{"n":2}"""); + Publish(invoices.Queue, """{"n":2}"""); + + await WaitAsync(async () => await CountAsync(orders.DocumentId) == 2, + TimeSpan.FromSeconds(60), "the active integration stopped working too"); + + // The deactivated one's message is still on the broker, waiting — not acknowledged and + // thrown away, which is the outcome that would look identical from Bitween's side. + await Task.Delay(TimeSpan.FromSeconds(3)); + Assert.Equal(1, await CountAsync(invoices.DocumentId)); + Assert.True(Depth(invoices.Queue) >= 1, + "the message for the deactivated gateway was consumed and dropped rather than left on the queue"); + } + + /// + /// Two queues, the same message id on each. Two messages, not a duplicate. + /// + /// A broker's message id is only unique within whatever produced it, so two systems publishing + /// to two queues on one broker can easily pick the same one. Deduplicating across the whole + /// data source would silently drop the second — so the key carries the endpoint. + /// + [Fact] + public async Task The_same_message_id_on_two_queues_is_two_messages() + { + var acme = await BrokerAsync("dedupe"); + var orders = await IntegrationAsync(acme, "orders"); + var invoices = await IntegrationAsync(acme, "invoices"); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + var shared = Guid.NewGuid().ToString("N"); + Publish(orders.Queue, """{"from":"orders"}""", shared); + Publish(invoices.Queue, """{"from":"invoices"}""", shared); + + await WaitAsync(async () => + await CountAsync(orders.DocumentId) == 1 && await CountAsync(invoices.DocumentId) == 1, + TimeSpan.FromSeconds(60), + "one of the two was deduplicated away — the key is not scoped to the endpoint"); + + // And a genuine redelivery on ONE of them still is a duplicate. + Publish(orders.Queue, """{"from":"orders"}""", shared); + await Task.Delay(TimeSpan.FromSeconds(5)); + Assert.Equal(1, await CountAsync(orders.DocumentId)); + } + + /// + /// Two data sources pointed at the same broker are two connections, owned independently. + /// + /// This is how a customer separates environments, or two of their own departments, on one + /// physical broker: the credentials, the health and the ownership are per data source, not per + /// broker, so one going down says nothing about the other. + /// + [Fact] + public async Task Two_data_sources_on_one_broker_are_owned_separately() + { + var acme = await BrokerAsync("tenant-a"); + var contoso = await BrokerAsync("tenant-b"); + + var acmeOrders = await IntegrationAsync(acme, "orders"); + var contosoOrders = await IntegrationAsync(contoso, "orders"); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + // One process each, not one shared: the connection belongs to the data source. + Assert.Single(Instances(acme)); + Assert.Single(Instances(contoso)); + Assert.NotEqual(Instances(acme)[0].ProcessId, Instances(contoso)[0].ProcessId); + + Publish(acmeOrders.Queue, """{"tenant":"acme"}"""); + Publish(contosoOrders.Queue, """{"tenant":"contoso"}"""); + + await WaitAsync(async () => + await CountAsync(acmeOrders.DocumentId) == 1 && + await CountAsync(contosoOrders.DocumentId) == 1, + TimeSpan.FromSeconds(60), "the two tenants did not both work"); + + // Ownership is per data source, so each holds its own lease. + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + foreach (var id in new[] { acme, contoso }) + Assert.NotNull(await db.Set().AsNoTracking() + .FirstOrDefaultAsync(l => l.Id == $"datasource.{id}")); + } + + /// + /// The point of all of it: three queues on one broker running three different integrations, + /// each producing its own result. + /// + /// Everything above is about plumbing. This is the outcome the plumbing exists for — a message + /// arriving on Acme's broker runs the subscription configured for that queue, and the result is + /// recorded against the right one. + /// + [Fact] + public async Task Each_queue_runs_its_own_integration_and_records_its_own_result() + { + var acme = await BrokerAsync("pipeline"); + var orders = await IntegrationAsync(acme, "orders", withSubscription: true); + var invoices = await IntegrationAsync(acme, "invoices", withSubscription: true); + + await using var supervisor = Supervisor(); + await supervisor.ReconcileAsync(); + + Publish(orders.Queue, """{"kind":"order","id":"A-1"}"""); + Publish(invoices.Queue, """{"kind":"invoice","id":"I-9"}"""); + + var orderXchange = await WaitForXchangeAsync(orders.DocumentId); + var invoiceXchange = await WaitForXchangeAsync(invoices.DocumentId); + + Assert.NotNull(orderXchange); + Assert.NotNull(invoiceXchange); + + // Each ran the subscription bound to ITS gateway, not the other's. + var orderResult = await ResultAsync(orderXchange!, orders.SubscriptionId); + var invoiceResult = await ResultAsync(invoiceXchange!, invoices.SubscriptionId); + + Assert.NotNull(orderResult); + Assert.NotNull(invoiceResult); + Assert.NotEqual(orders.SubscriptionId, invoices.SubscriptionId); + } + + // ---------------------------------------------------------------- helpers + + private static string Unique(string prefix) => $"{prefix}-{Guid.NewGuid():N}"[..24]; + + private sealed record Integration(string Queue, int DocumentId, int GatewayId, int SubscriptionId); + + /// A data source with connection settings only — the endpoints come from its gateways. + private async Task BrokerAsync(string label) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var dataSource = new DataSource + { + Name = Unique($"ds-{label}"), + AdapterId = BusAdapters.RabbitMq, + Kind = DataSourceKind.Broker, + Properties = new Dictionary(_fixture.ExternalRabbitProperties), + }; + + db.Add(dataSource); + await db.SaveChangesAsync(); + return dataSource.Id; + } + + /// One queue, one information type, one gateway — and optionally a subscription behind it. + private async Task IntegrationAsync(int dataSourceId, string label, + bool withSubscription = false) + { + var queue = Unique(label); + + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var document = new Document(null, Unique($"doc-{label}"), DocumentFormat.Json); + db.Add(document); + await db.SaveChangesAsync(); + + var subscriptionId = 0; + if (withSubscription) + { + // Every Subscription constructor sets Inactive = true, so it has to be turned on or + // the filter never matches it — see PipelineEndToEndTests. + var subscription = new Subscription(Unique($"sub-{label}"), document.Id, + SubscriptionType.BusGateway) + { + HandlerId = EchoHandler, + Inactive = false, + }; + subscription.SetDictionaries( + new Dictionary(), new Dictionary(), + new Dictionary(), new Dictionary(), + new Dictionary()); + + db.Add(subscription); + await db.SaveChangesAsync(); + subscriptionId = subscription.Id; + } + + var gateway = new BusGateway + { + Name = Unique($"gw-{label}"), + DocumentId = document.Id, + DataSourceId = dataSourceId, + Endpoint = queue, + }; + db.Add(gateway); + await db.SaveChangesAsync(); + + if (withSubscription) + { + db.Add(new BusGatewayRoute { BusGatewayId = gateway.Id, SubscriptionId = subscriptionId }); + await db.SaveChangesAsync(); + } + + // Ten-minute singleton snapshot: without revoking it the pipeline reads configuration from + // before this test's own setup and files the message against a null document. + scope.ServiceProvider.GetRequiredService().Revoke(); + + // Declared here rather than left to the adapter, so a queue exists to publish into even + // before the supervisor has started anything. + DeclareQueue(queue); + + return new Integration(queue, document.Id, gateway.Id, subscriptionId); + } + + private async Task SetGatewayInactiveAsync(int gatewayId, bool inactive) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var gateway = await db.Set().FirstAsync(g => g.Id == gatewayId); + gateway.Inactive = inactive; + await db.SaveChangesAsync(); + scope.ServiceProvider.GetRequiredService().Revoke(); + } + + private List Instances(int dataSourceId) => + _fixture.App.Services.GetRequiredService() + .Describe().Where(h => h.InstanceKey == dataSourceId.ToString()).ToList(); + + private async Task CountAsync(int documentId) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + return await db.Set().AsNoTracking() + .CountAsync(x => x.DocumentId == documentId && x.SubscriptionId == null); + } + + private async Task> ReferencesAsync(int documentId) + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + return await db.Set().AsNoTracking() + .Where(x => x.DocumentId == documentId && x.SubscriptionId == null) + .Select(x => x.Id) + .ToListAsync(); + } + + private async Task ResultAsync(Xchange xchange, int subscriptionId) + { + await ProcessAsync(xchange.Id); + + await using (var scope = _fixture.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var child = await db.Set().AsNoTracking() + .FirstOrDefaultAsync(x => x.DocumentId == xchange.DocumentId + && x.SubscriptionId == subscriptionId); + + Assert.True(child != null, + $"the arriving message did not produce a run for subscription {subscriptionId}"); + + await ProcessAsync(child!.Id); + + return await db.Set().AsNoTracking() + .FirstOrDefaultAsync(r => r.Id == child.Id); + } + } + + private async Task ProcessAsync(string xchangeId) + { + await using var scope = _fixture.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .Process("XchangeCreated", JsonConvert.SerializeObject(new { Id = xchangeId })); + } + + private async Task WaitForXchangeAsync(int documentId) + { + Xchange? found = null; + await WaitAsync(async () => + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + found = await db.Set().AsNoTracking() + .FirstOrDefaultAsync(x => x.DocumentId == documentId && x.SubscriptionId == null); + return found != null; + }, TimeSpan.FromSeconds(60), $"no Xchange was created for document {documentId}"); + + return found; + } + + private TestSupervisor Supervisor() => new( + new BusProviderSupervisor( + _fixture.App.Services, + _fixture.App.Services.GetRequiredService(), + new RabbitMqLeaderElection( + _fixture.App.Services.GetRequiredService(), + _fixture.App.Services, + _fixture.App.Services.GetRequiredService() + .CreateLogger()), + _fixture.App.Services.GetRequiredService() + .CreateLogger()), + _fixture.App.Services.GetRequiredService()); + + /// + /// A supervisor that cleans up after itself. Only instances that appeared after it was built + /// are stopped — the collection shares one resident host, so stopping "whatever is running" + /// would reach into another test. + /// + private sealed class TestSupervisor : IAsyncDisposable + { + private readonly BusProviderSupervisor _supervisor; + private readonly IResidentAdapterHost _adapters; + private readonly HashSet _preexisting; + private readonly HashSet _started = new(); + + public TestSupervisor(BusProviderSupervisor supervisor, IResidentAdapterHost adapters) + { + _supervisor = supervisor; + _adapters = adapters; + _preexisting = Keys(); + } + + private HashSet Keys() => + _adapters.Describe().Select(h => $"{h.AdapterId}|{h.InstanceKey}").ToHashSet(); + + public async Task ReconcileAsync() + { + await _supervisor.ReconcileAsync(); + foreach (var key in Keys().Where(k => !_preexisting.Contains(k))) _started.Add(key); + } + + public async ValueTask DisposeAsync() + { + try { await _supervisor.StopAsync(default); } catch { /* best effort */ } + + var live = Keys(); + foreach (var key in _started.Where(live.Contains)) + { + var parts = key.Split('|'); + try { await _adapters.StopAsync(parts[0], parts[1], drain: false); } catch { } + } + + _supervisor.Dispose(); + } + } + + private void Publish(string queue, string body, string? messageId = null) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + PublishOn(channel, queue, body, messageId); + } + + private static void PublishOn(IModel channel, string queue, string body, string? messageId = null) + { + channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false); + + var properties = channel.CreateBasicProperties(); + properties.ContentType = "application/json"; + properties.MessageId = messageId ?? Guid.NewGuid().ToString("N"); + properties.DeliveryMode = 2; + + channel.BasicPublish("", queue, properties, Encoding.UTF8.GetBytes(body)); + } + + private void DeclareQueue(string queue) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false); + } + + private uint Depth(string queue) + { + using var connection = ExternalConnection(); + using var channel = connection.CreateModel(); + return channel.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false).MessageCount; + } + + private IConnection ExternalConnection() => new ConnectionFactory + { + HostName = _fixture.ExternalRabbitHost, + Port = _fixture.ExternalRabbitPort, + UserName = _fixture.ExternalRabbitUser, + Password = _fixture.ExternalRabbitPassword, + }.CreateConnection("shared-broker-tests"); + + private static async Task WaitAsync(Func> condition, TimeSpan timeout, string because) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + try { if (await condition()) return; } catch { /* still settling */ } + await Task.Delay(300); + } + Assert.Fail($"Timed out after {timeout}: {because}"); + } +} diff --git a/SW.Bitween.Sdk/Model/BusGateway.cs b/SW.Bitween.Sdk/Model/BusGateway.cs index 9e103156..a0dd8885 100644 --- a/SW.Bitween.Sdk/Model/BusGateway.cs +++ b/SW.Bitween.Sdk/Model/BusGateway.cs @@ -10,6 +10,26 @@ public class BusGatewayCreate : IName /// Off but kept, with its routes. Messages stop reaching them. public bool Inactive { get; set; } + + /// + /// Null is the INTERNAL bus — the only behaviour that existed before, and still the + /// default. Set it and this gateway is fed by an external broker instead, through the + /// resident adapter that data source names. + /// + public int? DataSourceId { get; set; } + + /// + /// Which queue, topic or subscription on that data source feeds this gateway. Required for + /// an external gateway; meaningless for the internal bus, where the Document's own + /// BusMessageTypeName does the routing. + /// + public string Endpoint { get; set; } + + /// + /// Per-gateway overrides passed to the adapter. NOT YET CONSUMED by either bundled + /// adapter — see BusGateway.EndpointProperties on the entity. + /// + public Dictionary EndpointProperties { get; set; } = new(); } public class BusGatewayUpdate : BusGatewayCreate @@ -20,6 +40,12 @@ public class BusGatewayRow : BusGatewayUpdate { public int Id { get; set; } public string DocumentName { get; set; } + + /// Null for an internal gateway, which is what the list column reads. + public string DataSourceName { get; set; } + + /// Health of the connection behind it, so a broken broker is visible on the gateway. + public string DataSourceState { get; set; } public int? RoutesCount { get; set; } public ICollection Routes { get; set; } } diff --git a/SW.Bitween.Sdk/Model/DataSource.cs b/SW.Bitween.Sdk/Model/DataSource.cs new file mode 100644 index 00000000..d31b4536 --- /dev/null +++ b/SW.Bitween.Sdk/Model/DataSource.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; + +namespace SW.Bitween.Model; + +/// +/// How to reach an external system. A BusGateway with no data source still means the internal bus, +/// so setting one on a gateway is the single act that moves it onto a customer's broker. +/// +public class DataSourceCreate : IName +{ + public string Name { get; set; } + + /// The adapter that speaks this protocol, e.g. bitween.bus.rabbitmq. + public string AdapterId { get; set; } + + public string Kind { get; set; } = "Broker"; + + /// + /// Connection settings, handed to the adapter as startup values. Untyped on purpose: a provider + /// must not be limited to the subset of a broker's model Bitween happens to have modelled. + /// + public Dictionary Properties { get; set; } = new(); + + /// + /// Which names in hold credentials. Those come back from the API + /// masked, and a masked value saved again keeps whatever is already stored. + /// + public List SecretProperties { get; set; } = new(); + + public bool Inactive { get; set; } + + /// + /// How long a message's dedupe key is remembered. It has to exceed the widest redelivery window + /// this broker can produce. Zero turns deduplication off. + /// + public int DeduplicationWindowDays { get; set; } = 30; +} + +public class DataSourceUpdate : DataSourceCreate +{ +} + +public class DataSourceRow : DataSourceUpdate +{ + public int Id { get; set; } + + /// How many bus gateways this data source feeds. Deleting is refused while any do. + public int GatewayCount { get; set; } + + // ------------------------------------------------------------------ health + + public string LastKnownState { get; set; } + public DateTime? LastHeartbeatOn { get; set; } + public string LastException { get; set; } + public int ConsecutiveFailures { get; set; } + + /// Which node holds this connection, and at which fencing term. + public string OwnedByNode { get; set; } +} + +/// +/// Nothing to send: the data source already holds everything the test needs. It exists because a +/// keyed command takes a body, and an operator pressing Test is not supplying anything. +/// +public class DataSourceTestRequest +{ +} + +/// +/// What a Test button reports. Staged rather than a single boolean, because "it did not work" is +/// not an answer anyone can act on: the failing stage names what to go and fix. +/// +public class DataSourceTestResult +{ + public bool Succeeded { get; set; } + public string Error { get; set; } + + /// Stage name to outcome, in the order the adapter attempted them. + public List Stages { get; set; } = new(); + + /// Whatever the adapter chose to report — endpoint, queue depths, visibility timeout. + public Dictionary Details { get; set; } = new(); +} + +public class DataSourceTestStage +{ + public string Name { get; set; } + public bool Succeeded { get; set; } + public string Detail { get; set; } +} + +/// +/// What the connection is doing right now, as opposed to what it was configured to do. +/// +/// Read live from the resident adapter host rather than from the data source row, because the row +/// only carries what the last reconcile happened to write back — a summary, thirty seconds stale at +/// worst. This is the heartbeat itself: the counters the adapter keeps, the queue depths it can +/// see, and what the host observes about the process without needing the adapter's cooperation. +/// +public class DataSourceTelemetry +{ + /// + /// False when this node is not running the adapter. That is not a fault: a broker connection is + /// exclusive, so at most one node holds it and every other node answers this honestly rather + /// than reporting an outage it cannot see. + /// + public bool RunningHere { get; set; } + + /// Which node holds the connection, from the data source row — filled in even when it is not this one. + public string OwnedByNode { get; set; } + + // ---------------------------------------------------------------- adapter-reported + + public bool Connected { get; set; } + public string State { get; set; } + public DateTime? LastMessageOn { get; set; } + public long InFlight { get; set; } + public string LastError { get; set; } + + /// + /// Whatever the adapter chose to report: per-queue depth, messages received and acknowledged, + /// prefetch, the endpoint it is connected to. Untyped on purpose — Bitween does not model any + /// broker's telemetry any more than it models its topology. + /// + public Dictionary Details { get; set; } = new(); + + // ------------------------------------------------------- host-observed (no cooperation needed) + + /// These keep working when the adapter is wedged, which is exactly when they matter. + public int? ProcessId { get; set; } + public long WorkingSetBytes { get; set; } + public double CpuPercent { get; set; } + public int ThreadCount { get; set; } + public TimeSpan Uptime { get; set; } + public int RestartCount { get; set; } + public int MissedHeartbeats { get; set; } + + /// Restarted too many times too quickly, so the supervisor stopped trying. + public bool Quarantined { get; set; } + + public DateTime? LastHeartbeatOn { get; set; } + + /// What the adapter says it can do — the commands the UI could offer against it. + public List Commands { get; set; } = new(); +} diff --git a/SW.Bitween.Sdk/Model/Permissions.cs b/SW.Bitween.Sdk/Model/Permissions.cs index 2a31d394..827d913f 100644 --- a/SW.Bitween.Sdk/Model/Permissions.cs +++ b/SW.Bitween.Sdk/Model/Permissions.cs @@ -84,6 +84,17 @@ public static class BusGateways public const string Delete = "bus-gateways.delete"; } + public static class DataSources + { + public const string View = "data-sources.view"; + public const string Create = "data-sources.create"; + public const string Edit = "data-sources.edit"; + public const string Delete = "data-sources.delete"; + + /// Test a connection, which reaches out to the customer's broker. + public const string Operate = "data-sources.operate"; + } + public static class WorkGroups { public const string View = "workgroups.view"; @@ -224,6 +235,14 @@ private static PermissionAreaModel Area(string id, string label, string group, s (Delete, "Delete bus gateways.")), // ——— Configuration ——— + Area("data-sources", "Data sources", "Configuration", + "Connections to external brokers that bus gateways can read from.", + (View, "Browse data sources and their connection health."), + (Create, "Create data sources."), + (Edit, "Change connection settings and credentials."), + (Delete, "Delete data sources."), + (Operate, "Test a connection, which reaches out to the broker.")), + Area("workgroups", "Work groups", "Configuration", "Processing lanes that spread load across queues.", (View, "See work groups and their throughput."), (Create, "Create work groups."), diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index 670260ea..a6c5a031 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -10,6 +10,10 @@ import type { BusGateway, BusGatewayDetail, BusGatewayRow, + DataSourceDetail, + DataSourceRow, + DataSourceTelemetry, + DataSourceTestResult, DashboardData, ExchangeQuery, ExchangeRow, @@ -295,8 +299,47 @@ export interface ApiClient { }): Promise>; getBusGateway(id: number): Promise; createBusGateway(input: { name: string; informationTypeId: number }): Promise; - updateBusGateway(id: number, changes: { name: string; inactive: boolean }): Promise; + updateBusGateway( + id: number, + changes: { + name: string; + inactive: boolean; + /** Omit to leave the source alone; null moves the gateway back onto the internal bus. */ + dataSourceId?: number | null; + endpoint?: string | null; + }, + ): Promise; deleteBusGateway(id: number): Promise; + + // ——— Data sources ——— + listDataSources(): Promise; + searchDataSources(query: { + search: string; + offset: number; + limit: number; + }): Promise>; + getDataSource(id: number): Promise; + createDataSource(input: { + name: string; + adapterId: string; + properties: Record; + secretProperties: string[]; + }): Promise<{ id: number }>; + updateDataSource( + id: number, + changes: { + name: string; + adapterId: string; + kind: string; + properties: Record; + secretProperties: string[]; + inactive: boolean; + deduplicationWindowDays: number; + }, + ): Promise; + deleteDataSource(id: number): Promise; + testDataSource(id: number): Promise; + getDataSourceTelemetry(id: number): Promise; /** The subscription is either an existing id or defined inline; the endpoint commits both as one. */ addBusRoute(id: number, input: AddBusRouteInput): Promise; updateBusRoute( diff --git a/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts b/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts new file mode 100644 index 00000000..36065e40 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts @@ -0,0 +1,158 @@ +import type { ApiClient } from "../client"; +import type { + DataSourceDetail, + DataSourceRow, + DataSourceTelemetry, + DataSourceTestResult, + Paged, +} from "../types"; +import { get, post, request } from "./request"; + +interface SearchyResponse { + result: T[]; + totalCount: number; +} + +interface RawDataSource { + id: number; + name: string; + adapterId: string; + kind: string; + inactive: boolean | null; + deduplicationWindowDays: number; + gatewayCount: number; + lastKnownState: string | null; + lastHeartbeatOn: string | null; + lastException: string | null; + consecutiveFailures: number; + ownedByNode: string | null; + // Only the detail endpoint returns these — the list deliberately carries no connection + // settings at all, because even masked they are exposure with no purpose in a table. + properties?: Record | null; + secretProperties?: string[] | null; +} + +interface RawTestStage { + name: string; + succeeded: boolean; + detail: string | null; +} +interface RawTestResult { + succeeded: boolean; + error: string | null; + stages: RawTestStage[] | null; +} + +const toRow = (raw: RawDataSource): DataSourceRow => ({ + id: raw.id, + name: raw.name, + adapterId: raw.adapterId, + kind: raw.kind, + inactive: raw.inactive ?? false, + deduplicationWindowDays: raw.deduplicationWindowDays, + gatewayCount: raw.gatewayCount, + lastKnownState: raw.lastKnownState, + lastHeartbeatOn: raw.lastHeartbeatOn, + lastException: raw.lastException, + consecutiveFailures: raw.consecutiveFailures, + ownedByNode: raw.ownedByNode, +}); + +const toDetail = (raw: RawDataSource): DataSourceDetail => ({ + ...toRow(raw), + properties: raw.properties ?? {}, + secretProperties: raw.secretProperties ?? [], +}); + +// The backend's own default only applies when the caller omits the parameter, so asking for +// "everything" means naming a generously large number — same as the work groups module. +const EVERYTHING = 1_000_000; + +export const dataSourceMethods = { + async listDataSources(): Promise { + const res = await get>( + `/datasources?offset=0&limit=${EVERYTHING}`, + ); + return (res.result ?? []).map(toRow); + }, + + async searchDataSources(query: { + search: string; + offset: number; + limit: number; + }): Promise> { + const params = new URLSearchParams({ + offset: String(query.offset), + limit: String(query.limit), + }); + if (query.search.trim()) params.set("name", query.search.trim()); + + const res = await get>(`/datasources?${params.toString()}`); + return { total: res.totalCount, result: (res.result ?? []).map(toRow) }; + }, + + async getDataSource(id: number): Promise { + return toDetail(await get(`/datasources/${id}`)); + }, + + async createDataSource(input: { + name: string; + adapterId: string; + properties: Record; + secretProperties: string[]; + }): Promise<{ id: number }> { + const id = await post("/datasources", { + name: input.name, + adapterId: input.adapterId, + kind: "Broker", + properties: input.properties, + secretProperties: input.secretProperties, + inactive: false, + deduplicationWindowDays: 30, + }); + return { id }; + }, + + async updateDataSource( + id: number, + changes: { + name: string; + adapterId: string; + kind: string; + properties: Record; + secretProperties: string[]; + inactive: boolean; + deduplicationWindowDays: number; + }, + ): Promise { + await post(`/datasources/${id}`, changes); + }, + + async deleteDataSource(id: number): Promise { + await request(`/datasources/${id}`, { method: "DELETE" }); + }, + + /** Live, from the heartbeat. Scoped to the node that answers — see the backend handler. */ + async getDataSourceTelemetry(id: number): Promise { + const raw = await get(`/datasources/${id}/telemetry`); + return { ...raw, details: raw.details ?? {}, commands: raw.commands ?? [] }; + }, + + /** + * Reaches out to the broker and reports stage by stage. It runs the real adapter against the + * stored settings but never consumes, so the queues its gateways read are inspected rather than + * drained. + */ + async testDataSource(id: number): Promise { + const raw = await post(`/datasources/${id}/test`, {}); + return { + succeeded: raw.succeeded, + error: raw.error, + stages: (raw.stages ?? []).map((s) => ({ + name: s.name, + succeeded: s.succeeded, + detail: s.detail, + })), + }; + }, +} satisfies Partial; diff --git a/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts b/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts index fc6bcec5..ea2b88c3 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts @@ -54,6 +54,13 @@ interface RawBusGateway { routesCount: number | null; inactive: boolean | null; routes: RawBusGatewayRoute[] | null; + // Null dataSourceId is the internal bus. Optional rather than nullable because a bare POST + // response carries none of these. + dataSourceId?: number | null; + dataSourceName?: string | null; + dataSourceState?: string | null; + endpoint?: string | null; + endpointProperties?: Record | null; } const toApiGatewayAttachment = (p: RawApiGatewayPartner): ApiGatewayAttachment => ({ @@ -91,6 +98,14 @@ const toBusGatewayRoute = (r: RawBusGatewayRoute): BusGatewayRoute => ({ matchExpression: toMatchGroup(r.matchExpression), }); +/** Where the gateway reads from, shared by the row and the detail shapes. */ +const toSource = (raw: RawBusGateway) => ({ + dataSourceId: raw.dataSourceId ?? null, + dataSourceName: raw.dataSourceName ?? null, + dataSourceState: raw.dataSourceState ?? null, + endpoint: raw.endpoint ?? null, +}); + const toBusGatewayRow = (raw: RawBusGateway): BusGatewayRow => ({ id: raw.id, name: raw.name, @@ -100,6 +115,7 @@ const toBusGatewayRow = (raw: RawBusGateway): BusGatewayRow => ({ informationTypeCode: raw.documentName ?? "UNKNOWN", routeCount: raw.routesCount ?? raw.routes?.length ?? 0, routes: (raw.routes ?? []).map(toBusGatewayRoute), + ...toSource(raw), }); const toBusGatewayDetail = (raw: RawBusGateway): BusGatewayDetail => ({ @@ -111,6 +127,7 @@ const toBusGatewayDetail = (raw: RawBusGateway): BusGatewayDetail => ({ informationTypeCode: raw.documentName ?? "UNKNOWN", informationTypeName: raw.documentName ?? "Unknown", routes: (raw.routes ?? []).map(toBusGatewayRoute), + ...toSource(raw), }); /** The attachment always points at a subscription that already exists — a new one is @@ -261,21 +278,47 @@ export const gatewayMethods = { documentId: informationTypeId, inactive: false, }); - return { id, name, informationTypeId, inactive: false, createdOn: "" }; + // A gateway is created on the internal bus and moved onto a broker afterwards, on its own + // page — the source is a decision about an existing gateway, not a hurdle to creating one. + return { + id, + name, + informationTypeId, + inactive: false, + createdOn: "", + dataSourceId: null, + dataSourceName: null, + dataSourceState: null, + endpoint: null, + }; }, async updateBusGateway( id: number, - changes: { name: string; inactive: boolean }, + changes: { name: string; inactive: boolean; dataSourceId?: number | null; endpoint?: string | null }, ): Promise { // The bound information type is fixed at creation — Update.cs silently // ignores documentId — but the request DTO still requires a value, so // fetch the current one to round-trip it rather than sending a bogus 0. const current = await get(`/busgateways/${id}`); + + // The source is round-tripped the same way: a caller renaming the gateway must not silently + // move it back onto the internal bus by omitting the field. + const dataSourceId = + changes.dataSourceId !== undefined ? changes.dataSourceId : (current.dataSourceId ?? null); + const endpoint = + dataSourceId == null + ? null + : changes.endpoint !== undefined + ? changes.endpoint + : (current.endpoint ?? null); + await post(`/busgateways/${id}`, { name: changes.name, documentId: current.documentId, inactive: changes.inactive, + dataSourceId, + endpoint, }); return { id, @@ -283,6 +326,10 @@ export const gatewayMethods = { informationTypeId: current.documentId, inactive: changes.inactive, createdOn: "", + dataSourceId, + dataSourceName: dataSourceId === (current.dataSourceId ?? null) ? (current.dataSourceName ?? null) : null, + dataSourceState: null, + endpoint, }; }, diff --git a/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts b/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts index 8efb95e5..298c490a 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts @@ -2,6 +2,7 @@ import type { ApiClient } from "../client"; import { NotWiredError } from "../types"; import { adapterMethods } from "./adapters"; import { dashboardMethods } from "./dashboard"; +import { dataSourceMethods } from "./dataSources"; import { documentMethods } from "./documents"; import { exchangeMethods } from "./exchanges"; import { gatewayMethods } from "./gateways"; @@ -36,6 +37,7 @@ const wired: Partial = { ...exchangeMethods, ...queueHealthMethods, ...dashboardMethods, + ...dataSourceMethods, ...mapperMethods, ...notifierMethods, ...teamMethods, diff --git a/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts b/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts index 7587270f..40b36416 100644 --- a/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts +++ b/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts @@ -69,6 +69,15 @@ export const keys = { detail: (id: number | string) => ["bus-gateways", "detail", id] as const, }, + dataSources: { + all: ["data-sources"] as const, + list: ["data-sources", "list"] as const, + search: (params: Record) => ["data-sources", "search", params] as const, + detail: (id: number | string) => ["data-sources", "detail", id] as const, + /** Live heartbeat, polled — deliberately its own key so refreshing it never refetches the form. */ + telemetry: (id: number | string) => ["data-sources", "telemetry", id] as const, + }, + workGroups: { all: ["work-groups"] as const, list: ["work-groups", "list"] as const, diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index 258e96fe..26ec17ca 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -727,7 +727,109 @@ export interface BusGateway { /** Off but kept, with its routes. The message stops being offered to them. */ inactive: boolean; createdOn: string; + + /** + * Where the messages come from. Null is Bitween's own internal bus — the only behaviour that + * existed before data sources, and still the default, so no gateway changes meaning. Set it and + * this gateway is fed by a broker outside Bitween instead. + */ + dataSourceId: number | null; + dataSourceName: string | null; + /** Health of that connection, so a broker that has gone away shows on the gateway itself. */ + dataSourceState: string | null; + /** Which queue, topic or SQS URL on that data source feeds this gateway. */ + endpoint: string | null; +} + +// ——— Data sources ——— + +/** + * A connection to something outside Bitween — today a broker. + * + * It holds the connection and nothing else: what Bitween does with the messages belongs to the bus + * gateway pointing at it, which is why one data source can feed many gateways the way one broker + * connection serves many queues. + */ +export interface DataSource { + id: number; + name: string; + /** The adapter that speaks this protocol, e.g. bitween.bus.rabbitmq. */ + adapterId: string; + kind: string; + inactive: boolean; + /** + * How long a message's deduplication key is remembered. Has to exceed the widest redelivery + * window this broker can produce. Zero turns deduplication off. + */ + deduplicationWindowDays: number; + + // Health, written back by the supervisor from the adapter's heartbeat. + lastKnownState: string | null; + lastHeartbeatOn: string | null; + lastException: string | null; + consecutiveFailures: number; + /** Which node holds this connection, and at which fencing term. */ + ownedByNode: string | null; +} + +export interface DataSourceRow extends DataSource { + /** How many bus gateways read from it. Deleting is refused while any do. */ + gatewayCount: number; } + +export interface DataSourceDetail extends DataSourceRow { + /** Connection settings. Secret values arrive as the sentinel, never in clear. */ + properties: Record; + /** Which of those names hold credentials. */ + secretProperties: string[]; +} + +/** + * What the connection is doing right now, read from the adapter's heartbeat rather than from the + * data source row — which only carries what the last reconcile wrote back. + */ +export interface DataSourceTelemetry { + /** False when this node is not the one holding the connection. Not a fault: it is exclusive. */ + runningHere: boolean; + ownedByNode: string | null; + + connected: boolean; + state: string | null; + lastMessageOn: string | null; + inFlight: number; + lastError: string | null; + + /** Whatever the adapter reports — per-queue depth, counters, prefetch. Untyped by design. */ + details: Record; + + // Host-observed: these keep working when the adapter is wedged, which is when they matter. + processId: number | null; + workingSetBytes: number; + cpuPercent: number; + threadCount: number; + uptime: string; + restartCount: number; + missedHeartbeats: number; + quarantined: boolean; + lastHeartbeatOn: string | null; + + commands: string[]; +} + +export interface DataSourceTestStage { + name: string; + succeeded: boolean; + detail: string | null; +} + +export interface DataSourceTestResult { + succeeded: boolean; + error: string | null; + stages: DataSourceTestStage[]; +} + +/** Secret values come back as this. Saving it again keeps whatever is stored. */ +export const SECRET_SENTINEL = "__private__"; export interface BusGatewayRow extends BusGateway { informationTypeCode: string; routeCount: number; diff --git a/SW.Bitween.Web/ClientApp/src/nav.ts b/SW.Bitween.Web/ClientApp/src/nav.ts index 7ef6c18e..6eaf9a65 100644 --- a/SW.Bitween.Web/ClientApp/src/nav.ts +++ b/SW.Bitween.Web/ClientApp/src/nav.ts @@ -3,6 +3,7 @@ import { ArrowLeftRight, BellRing, Cable, + Database, CalendarClock, FileText, Handshake, @@ -62,6 +63,9 @@ export const NAV_GROUPS: NavGroup[] = [ items: [ { label: "API gateways", path: "/api-gateways", icon: Webhook, permissions: ["api-gateways.view"] }, { label: "Bus gateways", path: "/bus-gateways", icon: Cable, permissions: ["bus-gateways.view"] }, + // Directly under bus gateways: a data source is only ever reached through one, and the + // question it answers — "where do these messages come from?" — is a gateway's question. + { label: "Data sources", path: "/data-sources", icon: Database, permissions: ["data-sources.view"] }, { label: "Scheduled jobs", path: "/scheduled-jobs", icon: CalendarClock, permissions: ["subscriptions.view"] }, // Directly under scheduled jobs: it is the other thing that runs on a schedule, // and it collects what one of these produced. diff --git a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewayPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewayPage.tsx index acc5dac7..03138f88 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewayPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewayPage.tsx @@ -23,6 +23,8 @@ import { } from "./studio/Inspector"; import { PartnerDialog } from "../../components/config/PartnerDialog"; import { RouteList, type Selection } from "./studio/RouteList"; +import { SourceDialog } from "./SourceDialog"; +import { ConnectionBadge } from "../data-sources/ConnectionBadge"; import { BackLink } from "../../components/ui/BackLink"; import { keys } from "../../api/queryKeys"; import { @@ -107,6 +109,7 @@ export function BusGatewayPage() { const [removingRoute, setRemovingRoute] = useState(null); const [deletingGateway, setDeletingGateway] = useState(false); const [confirmingActive, setConfirmingActive] = useState(false); + const [editingSource, setEditingSource] = useState(false); /** A move the user asked for that would drop unsaved edits. */ const [guarded, setGuarded] = useState void }>(null); @@ -486,6 +489,8 @@ export function BusGatewayPage() { return (
+ {editingSource && setEditingSource(false)} />} + {/* ——— toolbar ——— */}
@@ -518,12 +523,20 @@ export function BusGatewayPage() { - {ownType && + {/* The bus message name, and the warning when there isn't one, apply only to a gateway + on the INTERNAL bus. An external one is fed by its data source's adapter and never + touches Bitween's own bus, so "not on the bus" would be a fault it does not have. */} + {g.dataSourceId == null && + ownType && (ownType.busMessageTypeName ? ( {ownType.busMessageTypeName} ) : ( @@ -533,6 +546,31 @@ export function BusGatewayPage() { ))} + {/* Where the messages come from. Next to the information type because the two are one + thought — what arrives, and from where — and both belong to the gateway rather than to + any one route. */} + + {/* With the list hidden there still has to be a way to reach route 94 of 127, and scrolling isn't it. */} {!listOpen && ( diff --git a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx index b3fbf4d2..c3eb4427 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx @@ -17,6 +17,7 @@ import { } from "../../components/config/shared"; import { matchSummary } from "../../lib/match"; import { keys } from "../../api/queryKeys"; +import { ConnectionBadge } from "../data-sources/ConnectionBadge"; /** * Bus gateways — messages picked off the bus. A gateway listens for one @@ -246,6 +247,22 @@ export function BusGatewaysPage() { /> ), }, + { + // Which bus this gateway actually reads. Worth a column: an operator should be able + // to see which of their gateways reach outside Bitween without opening each one. + header: "Source", + wrap: true, + cell: (g) => + g.dataSourceId == null ? ( + Internal bus + ) : ( +
+ {g.dataSourceName} + {g.endpoint} + +
+ ), + }, { // No Partners column here, unlike the API gateway list: a route's // partner is optional, so a gateway of unattributed routes would @@ -256,8 +273,13 @@ export function BusGatewaysPage() { // Nothing can reach this gateway at all if its type was taken off // the bus — no queue is declared for it. That outranks anything // the routes have to say. + // + // Only for a gateway on the INTERNAL bus, though: an external one is fed by its + // data source's adapter and never touches Bitween's own bus, so the information + // type's bus setting says nothing about whether messages arrive. const t = infoTypeById.get(g.informationTypeId); - if (t && !t.busEnabled) return Type not on bus; + if (g.dataSourceId == null && t && !t.busEnabled) + return Type not on bus; return ( void; +}) { + const queryClient = useQueryClient(); + + const [dataSourceId, setDataSourceId] = useState(gateway.dataSourceId); + const [endpoint, setEndpoint] = useState(gateway.endpoint ?? ""); + const [error, setError] = useState(null); + + const sources = useQuery({ + queryKey: keys.dataSources.list, + queryFn: () => api.listDataSources(), + }); + + const available = sources.data ?? []; + const selected = available.find((s) => s.id === dataSourceId); + + const save = useMutation({ + mutationFn: () => + api.updateBusGateway(gateway.id, { + name: gateway.name, + inactive: gateway.inactive, + dataSourceId, + // The endpoint goes with the source. Leaving one behind would show an internal gateway + // claiming to read a queue, which is the sort of thing that survives unnoticed for a year. + endpoint: dataSourceId == null ? null : endpoint.trim(), + }), + onSuccess: async () => { + await queryClient.invalidateQueries({ queryKey: keys.busGateways.all }); + onClose(); + }, + onError: (e) => + // The API refuses an external gateway with no endpoint, and one whose endpoint another + // gateway on the same data source already reads. Both are worth reading, not swallowing. + setError(e instanceof ApiRequestError ? e.message : "Could not change the source."), + }); + + return ( + +
+
+ + + +
+ + {available.length === 0 && ( + + No data sources exist yet, so there is no broker to point at.{" "} + + Add one first + + . + + )} + + {dataSourceId != null && ( + <> + + setAdapterId(e.target.value)} + options={BUS_PROVIDERS.map((p) => ({ value: p.id, label: p.label }))} + /> + + + {error && {error}} + +
+ + +
+
+
+ ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx new file mode 100644 index 00000000..c74635ef --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx @@ -0,0 +1,393 @@ +import { useEffect, useState } from "react"; +import { Link, useNavigate, useParams } from "react-router"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Check, Plug, Plus, Trash2, X } from "lucide-react"; +import { api, ApiRequestError, SECRET_SENTINEL, type DataSourceDetail, type DataSourceTestResult } from "../../api"; +import { Can, useSessionCan } from "../../auth/guards"; +import { PageHeader } from "../../components/layout/PageHeader"; +import { Badge, Button, FormError, LoadingBlock } from "../../components/ui/basics"; +import { Checkbox, Field, PasswordInput, TextInput } from "../../components/ui/forms"; +import { ConfirmDialog } from "../../components/ui/overlays"; +import { BackLink } from "../../components/ui/BackLink"; +import { keys } from "../../api/queryKeys"; +import { ConnectionBadge } from "./ConnectionBadge"; +import { isSecretName, providerOf } from "./providers"; +import { LiveConnection } from "./LiveConnection"; + +/** A draft is the whole editable surface, so the save bar can compare against what was loaded. */ +interface Draft { + name: string; + inactive: boolean; + deduplicationWindowDays: number; + properties: Record; +} + +const draftOf = (d: DataSourceDetail): Draft => ({ + name: d.name, + inactive: d.inactive, + deduplicationWindowDays: d.deduplicationWindowDays, + properties: { ...d.properties }, +}); + +/** + * One connection: its settings, whether it works, and who is holding it. + * + * The Test button carries more weight than it looks. Without it the only way to discover a wrong + * password is to save, wait up to thirty seconds for the supervisor to reconcile, and then read the + * health row — so a typo costs a minute and a guess at which of six fields caused it. + */ +export function DataSourcePage() { + const { id = "" } = useParams(); + const dataSourceId = Number(id); + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const canEdit = useSessionCan("data-sources.edit"); + + const source = useQuery({ + queryKey: keys.dataSources.detail(dataSourceId), + queryFn: () => api.getDataSource(dataSourceId), + retry: false, + refetchInterval: 10_000, + }); + + const [draft, setDraft] = useState(null); + const [error, setError] = useState(null); + const [result, setResult] = useState(null); + const [newKey, setNewKey] = useState(""); + const [removing, setRemoving] = useState(false); + + // Re-seed whenever the server's copy changes: saving re-masks the secrets, so the form has to + // go back to showing "stored" rather than a value the server will never return again. + useEffect(() => { + if (source.data) setDraft(draftOf(source.data)); + }, [source.data]); + + const save = useMutation({ + mutationFn: (d: Draft) => + api.updateDataSource(dataSourceId, { + name: d.name, + adapterId: source.data!.adapterId, + kind: source.data!.kind, + properties: d.properties, + secretProperties: source.data!.secretProperties, + inactive: d.inactive, + deduplicationWindowDays: d.deduplicationWindowDays, + }), + onSuccess: async () => { + setError(null); + await queryClient.invalidateQueries({ queryKey: keys.dataSources.all }); + }, + onError: (e) => setError(e instanceof ApiRequestError ? e.message : "Could not save."), + }); + + const test = useMutation({ + mutationFn: () => api.testDataSource(dataSourceId), + onSuccess: setResult, + onError: (e) => + setResult({ + succeeded: false, + error: e instanceof ApiRequestError ? e.message : "The test could not be run.", + stages: [], + }), + }); + + const remove = useMutation({ + mutationFn: () => api.deleteDataSource(dataSourceId), + onSuccess: async () => { + await queryClient.invalidateQueries({ queryKey: keys.dataSources.all }); + navigate("/data-sources"); + }, + onError: (e) => { + setRemoving(false); + setError(e instanceof ApiRequestError ? e.message : "Could not delete this data source."); + }, + }); + + if (source.isPending) return ; + if (source.isError || !source.data || !draft) + return This data source no longer exists.; + + const d = source.data; + const provider = providerOf(d.adapterId); + const dirty = JSON.stringify(draft) !== JSON.stringify(draftOf(d)); + + const setProperty = (key: string, value: string) => + setDraft({ ...draft, properties: { ...draft.properties, [key]: value } }); + + const removeProperty = (key: string) => { + const properties = { ...draft.properties }; + delete properties[key]; + setDraft({ ...draft, properties }); + }; + + const addProperty = () => { + const key = newKey.trim(); + if (!key || key in draft.properties) return; + setProperty(key, ""); + setNewKey(""); + }; + + return ( +
+ {removing && ( + setRemoving(false)} + onConfirm={() => remove.mutateAsync()} + body={ + d.gatewayCount > 0 + ? `${d.gatewayCount} bus gateway(s) still read from this connection, so deleting will be refused until they are moved off it.` + : "Its remembered deduplication keys go with it, so a message already processed could be handled again if it arrives later." + } + /> + )} + + + + + + + + + + +
+ } + /> + + {/* ——— what the last heartbeat said ——— */} +
+
+

Connection

+ +
+ +
+ {d.ownedByNode ?? "Not running on any node"} + + {d.lastHeartbeatOn ? new Date(d.lastHeartbeatOn).toLocaleString() : "—"} + + {String(d.consecutiveFailures)} + + + {d.gatewayCount} + + + {d.lastException && ( +
+
Last error
+
{d.lastException}
+
+ )} +
+ +

+ A broker connection is exclusive, so exactly one node holds it. The term after the node + name is the fencing token — it increases every time ownership moves, and a node whose term + is no longer current stops immediately rather than carrying on consuming. +

+
+ + {/* ——— what it is doing right now ——— */} + + + {/* ——— the test's answer ——— */} + {result && ( +
+
+

+ {result.succeeded ? "The connection works" : "The connection failed"} +

+ {result.succeeded ? "OK" : "Failed"} +
+ +
    + {result.stages.map((stage, i) => ( +
  • + {stage.succeeded ? ( + + ) : ( + + )} + {stage.name} + {stage.detail} +
    • + ))} +
+ + {!result.succeeded && result.error && ( +

{result.error}

+ )} + +

+ The test runs the real adapter against these settings but never consumes: the queues its + gateways read are inspected, not drained. +

+
+ )} + + {/* ——— settings ——— */} +
+

Settings

+ +
+ + setDraft({ ...draft, name: e.target.value })} + /> + + + setDraft({ ...draft, inactive: !e.target.checked })} + label="Active" + description="Turning this off stops the connection without losing its settings." + /> + + + + setDraft({ ...draft, deduplicationWindowDays: Number(e.target.value) || 0 }) + } + /> + + +
+

Connection settings

+

+ Handed straight to the adapter. Bitween does not model any broker's topology, so + anything the provider understands can go here. +

+ +
+ {Object.keys(draft.properties).map((key) => { + const secret = isSecretName(key, d.secretProperties); + const value = draft.properties[key]; + const stored = secret && value === SECRET_SENTINEL; + + return ( +
+
+ + {secret ? ( + setProperty(key, e.target.value)} + /> + ) : ( + setProperty(key, e.target.value)} + /> + )} + +
+ {canEdit && ( + + )} +
+ ); + })} +
+ + {canEdit && ( +
+
+ + setNewKey(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && addProperty()} + /> + +
+ +
+ )} +
+ + {error && {error}} + + {canEdit && ( +
+ + {dirty && ( + + )} + {!dirty && !save.isPending && No changes.} +
+ )} +
+
+
+ ); +} + +function Row({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+
{label}
+
{children}
+
+ ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcesPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcesPage.tsx new file mode 100644 index 00000000..7e40e7be --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcesPage.tsx @@ -0,0 +1,177 @@ +import { useNavigate, useSearchParams } from "react-router"; +import { keepPreviousData, useQuery } from "@tanstack/react-query"; +import { ArrowUpRight, Database, Plus, Search } from "lucide-react"; +import { api, type DataSourceRow } from "../../api"; +import { Can } from "../../auth/guards"; +import { PageHeader } from "../../components/layout/PageHeader"; +import { Button, EmptyState, LoadingBlock } from "../../components/ui/basics"; +import { Pagination } from "../../components/ui/Pagination"; +import { Table } from "../../components/ui/Table"; +import { keys } from "../../api/queryKeys"; +import { ConnectionBadge } from "./ConnectionBadge"; +import { providerLabel } from "./providers"; + +const PAGE_SIZE = 25; + +/** + * Connections to brokers outside Bitween. + * + * The health column is the reason this is a page rather than a dropdown on the gateway: a broker + * that has gone away is invisible everywhere else, and the node that holds each connection is the + * only place the exclusivity of a broker connection becomes visible at all. + */ +export function DataSourcesPage() { + const [searchParams, setSearchParams] = useSearchParams(); + const navigate = useNavigate(); + const q = searchParams.get("q") ?? ""; + const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : 0; + + const sources = useQuery({ + queryKey: keys.dataSources.search({ q, offset }), + queryFn: () => api.searchDataSources({ search: q, offset, limit: PAGE_SIZE }), + placeholderData: keepPreviousData, + // Health is written back by the supervisor on its own loop, so this is worth refreshing + // while someone watches a connection come up. + refetchInterval: 10_000, + }); + + const setParam = (key: string, value: string | null, resetOffset = true) => + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev); + if (value) next.set(key, value); + else next.delete(key); + if (resetOffset) next.delete("offset"); + return next; + }, + { replace: true }, + ); + + const rows = sources.data?.result ?? []; + const total = sources.data?.total ?? 0; + + return ( +
+ +

+ A data source holds a connection and nothing else. What Bitween + does with the messages belongs to the bus gateway pointing at it, which is why one + data source can feed many gateways — the same way one broker connection serves many + queues. +

+

+ A resident adapter holds the connection open and hands each message over. Nothing is + acknowledged to the broker until Bitween has persisted it, so a crash means + redelivery rather than a lost message — and every message carries a deduplication + key so the redelivery does not become a second exchange. +

+

+ A broker connection is exclusive: exactly one node may hold it. + Ownership is granted by a lease with a database-issued fencing term, so a node that + was paused while ownership moved discovers it and stops. +

+ + ), + }} + actions={ + + + + } + /> + +
+ + setParam("q", e.target.value || null)} + placeholder="Search data sources" + aria-label="Search data sources" + className="h-9 w-full rounded-lg border border-ink-200 bg-white pr-3 pl-9 text-sm placeholder:text-ink-400 focus:border-crimson-400 focus:ring-2 focus:ring-crimson-100 focus:outline-none" + /> +
+ + {sources.isPending ? ( + + ) : rows.length === 0 ? ( + } title={q ? "No data sources match" : "No data sources yet"}> + {q + ? "Try a different search." + : "A bus gateway with no data source reads Bitween's own internal bus. Add one here only to read a broker outside Bitween."} + + ) : ( + d.id} + minWidth="min-w-200" + onRowClick={(d) => navigate(`/data-sources/${d.id}`)} + footer={ + setParam("offset", String(o), false)} + /> + } + columns={[ + { + header: "Name", + cell: (d: DataSourceRow) => ( + + {d.name} + + ), + }, + { + header: "Provider", + cell: (d: DataSourceRow) => ( + {providerLabel(d.adapterId)} + ), + }, + { + header: "Connection", + wrap: true, + cell: (d: DataSourceRow) => ( +
+ + {d.lastException && ( + + {d.lastException} + + )} +
+ ), + }, + { + // A broker connection is exclusive, so which node holds it is not a detail — + // it is the answer to "why is this one quiet?". + header: "Held by", + cell: (d: DataSourceRow) => ( + {d.ownedByNode ?? "—"} + ), + }, + { + header: "Gateways", + align: "right", + cell: (d: DataSourceRow) => {d.gatewayCount}, + }, + { + header: "", + align: "right", + cell: () => , + }, + ]} + /> + )} + + ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/LiveConnection.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/LiveConnection.tsx new file mode 100644 index 00000000..77ec9a07 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/LiveConnection.tsx @@ -0,0 +1,171 @@ +import { useQuery } from "@tanstack/react-query"; +import { Activity } from "lucide-react"; +import { api } from "../../api"; +import { Badge, LoadingBlock } from "../../components/ui/basics"; +import { keys } from "../../api/queryKeys"; +import { ConnectionBadge } from "./ConnectionBadge"; + +/** + * What the connection is doing, as opposed to how it is configured. + * + * Two independent sources, kept visually apart because they fail differently. + * **Adapter-reported** — queue depths, messages handled — arrives on the heartbeat and stops + * arriving the moment the adapter wedges. **Host-observed** — memory, CPU, restarts — needs no + * cooperation from the adapter at all, so it still answers when the other half has gone quiet. + * That distinction is the whole reason an operator can tell "the broker is idle" from "the adapter + * is stuck", which no single health badge can say. + */ +export function LiveConnection({ dataSourceId }: { dataSourceId: number }) { + const telemetry = useQuery({ + queryKey: keys.dataSources.telemetry(dataSourceId), + queryFn: () => api.getDataSourceTelemetry(dataSourceId), + // Fast enough to watch a queue drain, slow enough not to be a load test of the heartbeat. + refetchInterval: 3_000, + retry: false, + }); + + if (telemetry.isPending) return ; + if (telemetry.isError || !telemetry.data) return null; + + const t = telemetry.data; + + // Queue depths come through as depth:, which is the one part of the adapter's untyped + // detail bag worth promoting: backlog per queue is the number an operator actually watches. + const depths = Object.entries(t.details) + .filter(([k]) => k.startsWith("depth:")) + .map(([k, v]) => [k.slice("depth:".length), v] as const); + + const counters = ["received", "acked", "nacked", "failed", "deleted", "returned", "sent"] + .filter((k) => k in t.details) + .map((k) => [k, t.details[k]] as const); + + const rest = Object.entries(t.details).filter( + ([k]) => !k.startsWith("depth:") && !counters.some(([c]) => c === k), + ); + + return ( +
+
+ +

Live

+ + {t.quarantined && ( + + Quarantined + + )} + {t.missedHeartbeats > 0 && ( + + {t.missedHeartbeats} missed heartbeat{t.missedHeartbeats === 1 ? "" : "s"} + + )} +
+ + {!t.runningHere ? ( +

+ {t.ownedByNode + ? `Held by ${t.ownedByNode}, which is not this node — a broker connection is exclusive, so only that node can see its live figures.` + : "Not running on any node. External bus providers are opt-in per node, so this is a setting rather than a fault."} +

+ ) : ( +
+ {depths.length > 0 && ( +
+

+ Backlog per queue +

+
+ {depths.map(([queue, depth]) => ( + + {queue} + {depth} + + ))} +
+
+ )} + + {counters.length > 0 && ( +
+

+ Adapter-reported +

+
+ {counters.map(([name, value]) => ( + + ))} + + {t.lastMessageOn && ( + + )} +
+
+ )} + +
+

+ Host-observed +

+

+ Measured from outside the adapter, so these still answer when it has stopped + reporting. +

+
+ + + + + + +
+
+ + {rest.length > 0 && ( +
+ + Everything else the adapter reports + +
+ {rest.map(([k, v]) => ( +
+
{k}
+
+ {v} +
+
+ ))} +
+
+ )} + + {t.lastError &&

{t.lastError}

} +
+ )} +
+ ); +} + +function Stat({ label, value }: { label: string; value: string }) { + return ( + + {label} + {value} + + ); +} + +const mb = (bytes: number) => (bytes > 0 ? `${(bytes / 1024 / 1024).toFixed(0)} MB` : "—"); + +/** The backend serializes a TimeSpan as "d.hh:mm:ss.fffffff" or "hh:mm:ss.fffffff". */ +const duration = (value: string): string => { + const m = /^(?:(\d+)\.)?(\d{2}):(\d{2}):(\d{2})/.exec(value ?? ""); + if (!m) return "—"; + const [, d, hh, mm, ss] = m; + if (d && Number(d) > 0) return `${d}d ${Number(hh)}h`; + if (Number(hh) > 0) return `${Number(hh)}h ${Number(mm)}m`; + if (Number(mm) > 0) return `${Number(mm)}m ${Number(ss)}s`; + return `${Number(ss)}s`; +}; diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/providers.ts b/SW.Bitween.Web/ClientApp/src/pages/data-sources/providers.ts new file mode 100644 index 00000000..ce4516b5 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/providers.ts @@ -0,0 +1,78 @@ +/** + * The bus providers Bitween ships, and what each expects. + * + * Connection settings are untyped by design — a provider must not be limited to the subset of a + * broker's model Bitween happens to have modelled — but an empty key/value grid is not a form + * anyone can fill in. This is presentation only: it decides which fields a new data source starts + * with, not which ones the adapter will accept. + */ +export interface BusProvider { + id: string; + label: string; + description: string; + /** Field name to a short explanation, shown under the input. */ + hints: Record; + defaults: Record; + secrets: string[]; +} + +export const BUS_PROVIDERS: BusProvider[] = [ + { + id: "bitween.bus.rabbitmq", + label: "RabbitMQ", + description: "An AMQP broker the customer runs, separate from Bitween's own bus.", + defaults: { + Host: "", + Port: "5672", + UserName: "", + Password: "", + VirtualHost: "/", + DeclareMode: "assert", + Prefetch: "16", + }, + secrets: ["Password"], + hints: { + DeclareMode: + "none — assume everything exists. assert — check and fail loudly if not. create — declare the queues.", + Prefetch: "How many messages the broker lets Bitween hold unacknowledged at once.", + Tls: "Set to true for anything that is not localhost — AMQP authenticates in the clear otherwise.", + }, + }, + { + id: "bitween.bus.sqs", + label: "Amazon SQS", + description: + "An SQS queue, including the one an Amazon Selling Partner notification subscription delivers to.", + defaults: { + Region: "eu-west-1", + AccessKeyId: "", + SecretAccessKey: "", + WaitTimeSeconds: "20", + VisibilityTimeoutSeconds: "60", + UnwrapSellingPartnerNotification: "false", + }, + secrets: ["AccessKeyId", "SecretAccessKey"], + hints: { + AccessKeyId: "Leave both keys blank on AWS to use the instance profile or IRSA instead.", + VisibilityTimeoutSeconds: + "Must exceed how long Bitween takes to persist a message, or SQS redelivers one already being handled.", + UnwrapSellingPartnerNotification: + "true unwraps the SP-API envelope so subscriptions see the notification payload itself.", + }, + }, +]; + +export const providerOf = (adapterId: string): BusProvider | undefined => + BUS_PROVIDERS.find((p) => p.id === adapterId); + +export const providerLabel = (adapterId: string): string => + providerOf(adapterId)?.label ?? adapterId; + +/** + * Whether a setting holds a credential. The backend decides this for real — and masks + * accordingly — but the form needs to know before a value has ever been saved. + */ +const CREDENTIAL = /password|secret|token|credential|apikey|accesskey|privatekey|connectionstring|sas|passphrase|certificate/i; + +export const isSecretName = (name: string, declared: string[] = []): boolean => + declared.some((d) => d.toLowerCase() === name.toLowerCase()) || CREDENTIAL.test(name); diff --git a/SW.Bitween.Web/ClientApp/src/router.tsx b/SW.Bitween.Web/ClientApp/src/router.tsx index 5fe93ac8..59a352b2 100644 --- a/SW.Bitween.Web/ClientApp/src/router.tsx +++ b/SW.Bitween.Web/ClientApp/src/router.tsx @@ -22,6 +22,9 @@ import { EditAttachmentPage } from "./pages/api-gateways/EditAttachmentPage"; import { BusGatewayNewPage } from "./pages/bus-gateways/BusGatewayNewPage"; import { BusGatewayPage } from "./pages/bus-gateways/BusGatewayPage"; import { BusGatewaysPage } from "./pages/bus-gateways/BusGatewaysPage"; +import { DataSourceNewPage } from "./pages/data-sources/DataSourceNewPage"; +import { DataSourcePage } from "./pages/data-sources/DataSourcePage"; +import { DataSourcesPage } from "./pages/data-sources/DataSourcesPage"; import { FlowPage } from "./pages/flow/FlowPage"; import { GlobalValueSetPage } from "./pages/global-values/GlobalValueSetPage"; import { GlobalValueSetsPage } from "./pages/global-values/GlobalValueSetsPage"; @@ -201,6 +204,14 @@ export const router = createBrowserRouter([ ), }, + { + path: "data-sources", + element: ( + + + + ), + }, { path: "flow", element: ( @@ -289,6 +300,23 @@ export const router = createBrowserRouter([ ), }, + { + // Before the :id route, or "new" is read as an id. + path: "data-sources/new", + element: ( + + + + ), + }, + { + path: "data-sources/:id", + element: ( + + + + ), + }, { path: "scheduled-jobs/new", element: ( From fcc9efbdf998520dffcd6a19f87f6c6b3d42e3e7 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Sun, 6 Sep 2026 17:26:57 +0300 Subject: [PATCH 11/43] feat: memory and CPU ceilings for bus adapters, on SimplyWorks.Serverless 8.1.16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adapter is a separate process holding a customer's broker connection, and Bitween constrained it in no way at all: one runaway payload was bounded only by the host, taking every other integration on the node with it. The runtime had supported memory ceilings all along and nothing passed them; CPU had no ceiling to pass. Both are now configurable per data source, from the UI. 217 unit + 265 integration + 90 ClientApp tests pass. Packages - SimplyWorks.Serverless[.Sdk] 8.1.14 -> 8.1.16 across all nine references, after checking the published assembly actually carries ResourceLimits, CpuPercentLimit, UpdateLimitsAsync, RestartAsync and CommandDetails rather than assuming the version number implied them. Ceilings - DataSource gains SoftMemoryLimitMb, HardMemoryLimitMb, CpuPercentLimit and CpuLimitSamples, migrated on all three providers. - The supervisor passes them into the AdapterSpec AND folds them into the fingerprint. Without the second half a raised limit would save cleanly, change nothing, and the adapter would keep running under the old one — with nothing anywhere to say it had not taken. - Guards that refuse a combination which can never fire: a soft memory ceiling above the hard one (the runtime fails the allocation first, so the graceful recycle never happens), and a CPU ceiling above 100% (the figure is a share of the whole node, so nothing can exceed it). Enforced in the handler, not only in the FluentValidation validator, because the validator runs in the HTTP pipeline and nothing else does. UI - Memory ceilings and a CPU ceiling on the data source page, each saying what crossing it actually does — drain versus kill — since that is the difference between messages going back to the broker and being lost. - The CPU copy carries the warning that the figure is a share of the WHOLE node rather than of one core: one core pegged on a sixteen-core node reads about 6%, so "50%" would allow eight cores. That misreading is not hypothetical — it broke the first version of the serverless test. The "for scale" hint says outright that it uses the browser's core count, not the node's, rather than implying knowledge of the server. Also adds /datasources/{id}/inspect, relaying Discover and GetStats to the adapter actually serving traffic — not a throwaway instance, the way the connection test does, because these are questions about the live connection. An allow-list rather than a passthrough: the adapter also exposes Publish, which writes to the customer's broker, and this endpoint is guarded by View. Tests - Changing either ceiling restarts the adapter (mutation-verified: removing the ceilings from the fingerprint makes it fail), both refusals, and the not-running answers for inspect. - Fixed a leak in the test helper that was the real cause of seven unrelated failures: it created a RabbitMqLeaderElection per test and never disposed it, so the exclusive queue that IS the lock stayed held and later reconciles found every data source owned by a node that no longer existed. Co-Authored-By: Claude Opus 5 --- .../SW.Bitween.Adapters.Bus.RabbitMq.csproj | 2 +- .../SW.Bitween.Adapters.Bus.Sqs.csproj | 2 +- .../Domain/DataSources/DataSource.cs | 37 + .../Resources/DataSources/Create.cs | 50 +- SW.Bitween.Api/Resources/DataSources/Get.cs | 4 + .../Resources/DataSources/Inspect.cs | 93 + .../Resources/DataSources/Search.cs | 4 + .../Resources/DataSources/Update.cs | 14 + SW.Bitween.Api/SW.Bitween.Api.csproj | 2 +- .../DataSources/BusProviderSupervisor.cs | 19 +- .../SW.Bitween.IntegrationTests.csproj | 2 +- .../Tests/DataSourceApiTests.cs | 215 +- ...0906132336_AdapterMemoryLimits.Designer.cs | 2296 ++++++++++++++ .../20260906132336_AdapterMemoryLimits.cs | 40 + ...0260906141722_AdapterCpuLimits.Designer.cs | 2302 ++++++++++++++ .../20260906141722_AdapterCpuLimits.cs | 40 + .../BitweenDbContextModelSnapshot.cs | 12 + ...0906132331_AdapterMemoryLimits.Designer.cs | 2289 ++++++++++++++ .../20260906132331_AdapterMemoryLimits.cs | 40 + ...0260906141717_AdapterCpuLimits.Designer.cs | 2295 ++++++++++++++ .../20260906141717_AdapterCpuLimits.cs | 40 + .../BitweenDbContextModelSnapshot.cs | 12 + ...0906132326_AdapterMemoryLimits.Designer.cs | 2633 ++++++++++++++++ .../20260906132326_AdapterMemoryLimits.cs | 44 + ...0260906141712_AdapterCpuLimits.Designer.cs | 2641 +++++++++++++++++ .../20260906141712_AdapterCpuLimits.cs | 44 + .../BitweenDbContextModelSnapshot.cs | 16 + ...W.Bitween.SampleConfigurableAdapter.csproj | 2 +- .../SW.Bitween.SampleHandler.csproj | 2 +- .../SW.Bitween.SampleMapper.csproj | 2 +- .../SW.Bitween.SampleValidator.csproj | 2 +- SW.Bitween.Sdk/Model/DataSource.cs | 44 + SW.Bitween.Web/ClientApp/src/api/client.ts | 7 + .../ClientApp/src/api/http/dataSources.ts | 21 + SW.Bitween.Web/ClientApp/src/api/types.ts | 22 + .../src/pages/data-sources/DataSourcePage.tsx | 210 +- SW.Bitween.Web/SW.Bitween.Web.csproj | 2 +- 37 files changed, 15488 insertions(+), 14 deletions(-) create mode 100644 SW.Bitween.Api/Resources/DataSources/Inspect.cs create mode 100644 SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.Designer.cs create mode 100644 SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.cs create mode 100644 SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.Designer.cs create mode 100644 SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.cs create mode 100644 SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.Designer.cs create mode 100644 SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.cs create mode 100644 SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.Designer.cs create mode 100644 SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.cs create mode 100644 SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.Designer.cs create mode 100644 SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.cs create mode 100644 SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.Designer.cs create mode 100644 SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.cs diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj index 13ad6ebd..6d02ff56 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj +++ b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj @@ -9,6 +9,6 @@ - + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj index f97caada..af659718 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj +++ b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj @@ -7,6 +7,6 @@ - + diff --git a/SW.Bitween.Api/Domain/DataSources/DataSource.cs b/SW.Bitween.Api/Domain/DataSources/DataSource.cs index 72ba73c5..b2011ef5 100644 --- a/SW.Bitween.Api/Domain/DataSources/DataSource.cs +++ b/SW.Bitween.Api/Domain/DataSources/DataSource.cs @@ -55,6 +55,43 @@ public class DataSource : BaseEntity, IAudited /// public int DeduplicationWindowDays { get; set; } = 30; + /// + /// A soft ceiling in megabytes. Crossing it is a signal, not a kill: the host reports it and + /// the adapter is recycled between messages, so nothing in flight is lost. Zero leaves the + /// host's own default in place. + /// + public int SoftMemoryLimitMb { get; set; } + + /// + /// A hard ceiling in megabytes, enforced by the runtime rather than by the supervisor's + /// goodwill — it becomes the adapter process's GC heap hard limit, so an allocation past it + /// fails inside the adapter instead of taking the node down with it. Zero leaves the host's + /// own default in place. + /// + /// This matters because an adapter is a separate process holding a broker connection: without + /// a ceiling, one customer's runaway payload is bounded by nothing but the host, and every + /// other integration on the node goes down with it. + /// + public int HardMemoryLimitMb { get; set; } + + /// + /// Sustained CPU ceiling for the adapter process, as a percentage of the WHOLE node — the same + /// figure the heartbeat reports. Worth being exact about, because the intuitive reading is + /// wrong in an expensive direction: one core pegged flat out on a sixteen-core node reads about + /// 6%, so a ceiling set at "50%, surely that's half a core" would in fact allow eight. + /// + /// Deliberately sustained rather than instantaneous — an adapter draining a backlog is supposed + /// to work hard, and recycling it for that would be a bug wearing a limit's clothes. Zero + /// leaves the host default in place. + /// + public double CpuPercentLimit { get; set; } + + /// + /// How many consecutive heartbeats above before it trips. Zero + /// uses the host default. Longer means a bigger burst of legitimate work passes underneath it. + /// + public int CpuLimitSamples { get; set; } + // ---------------------------------------------------------------- health /// Last state the adapter reported on its heartbeat: Connected, Idle, Disconnected... diff --git a/SW.Bitween.Api/Resources/DataSources/Create.cs b/SW.Bitween.Api/Resources/DataSources/Create.cs index 5120afa6..89035812 100644 --- a/SW.Bitween.Api/Resources/DataSources/Create.cs +++ b/SW.Bitween.Api/Resources/DataSources/Create.cs @@ -23,6 +23,9 @@ public async Task Handle(DataSourceCreate model) { await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.Create); + EnsureCeilingsAreUsable(model.SoftMemoryLimitMb, model.HardMemoryLimitMb, + model.CpuPercentLimit, model.CpuLimitSamples); + var nameTaken = await _dbContext.Set() .AnyAsync(d => d.Name == model.Name); if (nameTaken) @@ -40,7 +43,11 @@ public async Task Handle(DataSourceCreate model) Properties = properties, SecretProperties = Secrets.Declare(properties, model.SecretProperties), Inactive = model.Inactive, - DeduplicationWindowDays = model.DeduplicationWindowDays + DeduplicationWindowDays = model.DeduplicationWindowDays, + SoftMemoryLimitMb = model.SoftMemoryLimitMb, + HardMemoryLimitMb = model.HardMemoryLimitMb, + CpuPercentLimit = model.CpuPercentLimit, + CpuLimitSamples = model.CpuLimitSamples }; _dbContext.Add(entity); @@ -48,6 +55,35 @@ public async Task Handle(DataSourceCreate model) return entity.Id; } + /// + /// Enforced here rather than only in the validator, because the validator runs in the HTTP + /// pipeline and nothing else does: a caller reaching the handler another way — the supervisor's + /// own tests, a future internal caller — would otherwise save a combination that can never + /// work. A soft ceiling above the hard one is unreachable: the runtime fails the allocation + /// first, so the graceful recycle it was configured for never happens. + /// + internal static void EnsureCeilingsAreUsable(int softMb, int hardMb, + double cpuPercent = 0, int cpuSamples = 0) + { + if (softMb < 0 || hardMb < 0) + throw new SWException("A memory ceiling cannot be negative. Use 0 for the host default."); + + if (cpuPercent < 0 || cpuSamples < 0) + throw new SWException("A CPU ceiling cannot be negative. Use 0 for the host default."); + + // Above 100 is not a ceiling, because the figure is a share of the whole node — nothing can + // ever exceed it, so the limit would silently never fire. + if (cpuPercent > 100) + throw new SWException( + $"A CPU ceiling of {cpuPercent}% can never be reached: the figure is a share of the " + + "whole node, so 100% is every core at once."); + + if (softMb > 0 && hardMb > 0 && softMb > hardMb) + throw new SWException( + $"The soft memory limit ({softMb} MB) has to be at or below the hard limit " + + $"({hardMb} MB), or it can never be reached."); + } + internal static DataSourceKind ParseKind(string kind) => Enum.TryParse(kind, ignoreCase: true, out var parsed) ? parsed @@ -62,6 +98,18 @@ public Validate() // Zero is meaningful — it turns deduplication off — so only a negative is rejected. RuleFor(i => i.DeduplicationWindowDays).GreaterThanOrEqualTo(0); + + // Zero means "leave the host default alone" for both. + RuleFor(i => i.SoftMemoryLimitMb).GreaterThanOrEqualTo(0); + RuleFor(i => i.HardMemoryLimitMb).GreaterThanOrEqualTo(0); + + // A soft ceiling above the hard one can never be reached: the runtime fails the + // allocation first, so the recycle it was meant to trigger never happens. + RuleFor(i => i.SoftMemoryLimitMb) + .LessThanOrEqualTo(i => i.HardMemoryLimitMb) + .When(i => i.SoftMemoryLimitMb > 0 && i.HardMemoryLimitMb > 0) + .WithMessage("The soft memory limit has to be at or below the hard limit, " + + "or it can never be reached."); } } } diff --git a/SW.Bitween.Api/Resources/DataSources/Get.cs b/SW.Bitween.Api/Resources/DataSources/Get.cs index 1709b14e..f99ee6b7 100644 --- a/SW.Bitween.Api/Resources/DataSources/Get.cs +++ b/SW.Bitween.Api/Resources/DataSources/Get.cs @@ -38,6 +38,10 @@ public async Task Handle(int key) Kind = dataSource.Kind.ToString(), Inactive = dataSource.Inactive, DeduplicationWindowDays = dataSource.DeduplicationWindowDays, + SoftMemoryLimitMb = dataSource.SoftMemoryLimitMb, + CpuPercentLimit = dataSource.CpuPercentLimit, + CpuLimitSamples = dataSource.CpuLimitSamples, + HardMemoryLimitMb = dataSource.HardMemoryLimitMb, // Masked, always. This is the only endpoint that returns connection settings, so it is // the only place a broker password could leave the process. diff --git a/SW.Bitween.Api/Resources/DataSources/Inspect.cs b/SW.Bitween.Api/Resources/DataSources/Inspect.cs new file mode 100644 index 00000000..ba84c429 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Inspect.cs @@ -0,0 +1,93 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; + +namespace SW.Bitween.Resources.DataSources; + +/// +/// Asks the RUNNING adapter what it can see on the broker right now. +/// +/// Deliberately not the same shape as . A test starts a throwaway instance, +/// because its whole job is to answer "would these settings work" before anything depends on them. +/// Discover and GetStats are questions about the connection that is actually serving traffic, so +/// they go to that instance — starting a second one would answer about a connection nobody is +/// using, and on a broker that authorises per-connection it might not even answer the same way. +/// +/// Both are read-only: neither consumes, acknowledges or publishes anything. +/// +[HandlerName("inspect")] +public class Inspect : ICommandHandler +{ + /// + /// The commands this endpoint will relay. An allow-list rather than a passthrough: the + /// adapter also exposes Publish, which writes to the customer's broker, and that is not + /// something a View-level read should be able to reach by naming it in a request body. + /// + private static readonly string[] Allowed = ["Discover", "GetStats"]; + + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + private readonly IResidentAdapterHost _adapters; + + public Inspect(BitweenDbContext dbContext, RequestContext requestContext, + IResidentAdapterHost adapters = null) + { + _dbContext = dbContext; + _requestContext = requestContext; + _adapters = adapters; + } + + public async Task Handle(int key, DataSourceInspectRequest request) + { + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.View); + + var command = request?.Command ?? "Discover"; + if (!Allowed.Contains(command, StringComparer.OrdinalIgnoreCase)) + throw new SWException( + $"'{command}' is not something this endpoint relays. Allowed: {string.Join(", ", Allowed)}."); + + var exists = await _dbContext.Set().AsNoTracking().AnyAsync(d => d.Id == key); + if (!exists) + throw new SWNotFoundException($"DataSource with id '{key}' was not found"); + + var instance = _adapters?.Describe() + .FirstOrDefault(h => h.InstanceKey == key.ToString()); + + if (instance == null) + return new DataSourceInspectResult + { + Ran = false, + // A broker connection is exclusive, so "not here" is the normal answer on every + // node but one — not a fault, and worth saying in those words. + Error = "This node is not running the adapter for this data source, so it has " + + "nothing to ask. A broker connection is exclusive: only the node holding " + + "it can answer.", + }; + + try + { + var live = _adapters.Get(instance.AdapterId, instance.InstanceKey); + if (live == null) + return new DataSourceInspectResult { Ran = false, Error = "The adapter went away." }; + + var raw = await live.InvokeAsync(command, timeoutSeconds: 30); + return new DataSourceInspectResult + { + Ran = true, + Command = command, + Result = raw?.ToString(Newtonsoft.Json.Formatting.Indented) ?? "", + }; + } + catch (Exception ex) + { + // A command that fails is an answer about the broker, not a server error. + return new DataSourceInspectResult { Ran = false, Command = command, Error = ex.Message }; + } + } +} diff --git a/SW.Bitween.Api/Resources/DataSources/Search.cs b/SW.Bitween.Api/Resources/DataSources/Search.cs index 2f4323b6..05b0b800 100644 --- a/SW.Bitween.Api/Resources/DataSources/Search.cs +++ b/SW.Bitween.Api/Resources/DataSources/Search.cs @@ -36,6 +36,10 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa Kind = dataSource.Kind.ToString(), Inactive = dataSource.Inactive, DeduplicationWindowDays = dataSource.DeduplicationWindowDays, + SoftMemoryLimitMb = dataSource.SoftMemoryLimitMb, + CpuPercentLimit = dataSource.CpuPercentLimit, + CpuLimitSamples = dataSource.CpuLimitSamples, + HardMemoryLimitMb = dataSource.HardMemoryLimitMb, LastKnownState = dataSource.LastKnownState, LastHeartbeatOn = dataSource.LastHeartbeatOn, LastException = dataSource.LastException, diff --git a/SW.Bitween.Api/Resources/DataSources/Update.cs b/SW.Bitween.Api/Resources/DataSources/Update.cs index d157a490..31198be8 100644 --- a/SW.Bitween.Api/Resources/DataSources/Update.cs +++ b/SW.Bitween.Api/Resources/DataSources/Update.cs @@ -22,6 +22,9 @@ public async Task Handle(int key, DataSourceUpdate model) { await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.Edit); + Create.EnsureCeilingsAreUsable(model.SoftMemoryLimitMb, model.HardMemoryLimitMb, + model.CpuPercentLimit, model.CpuLimitSamples); + var entity = await _dbContext.Set().FirstOrDefaultAsync(d => d.Id == key); if (entity == null) throw new SWNotFoundException($"DataSource with Id {key} not found"); @@ -42,6 +45,10 @@ public async Task Handle(int key, DataSourceUpdate model) entity.SecretProperties = Secrets.Declare(properties, model.SecretProperties); entity.Inactive = model.Inactive; entity.DeduplicationWindowDays = model.DeduplicationWindowDays; + entity.SoftMemoryLimitMb = model.SoftMemoryLimitMb; + entity.HardMemoryLimitMb = model.HardMemoryLimitMb; + entity.CpuPercentLimit = model.CpuPercentLimit; + entity.CpuLimitSamples = model.CpuLimitSamples; await _dbContext.SaveChangesAsync(); @@ -57,6 +64,13 @@ public Validate() RuleFor(i => i.Name).NotEmpty().MaximumLength(200); RuleFor(i => i.AdapterId).NotEmpty().MaximumLength(200); RuleFor(i => i.DeduplicationWindowDays).GreaterThanOrEqualTo(0); + RuleFor(i => i.SoftMemoryLimitMb).GreaterThanOrEqualTo(0); + RuleFor(i => i.HardMemoryLimitMb).GreaterThanOrEqualTo(0); + RuleFor(i => i.SoftMemoryLimitMb) + .LessThanOrEqualTo(i => i.HardMemoryLimitMb) + .When(i => i.SoftMemoryLimitMb > 0 && i.HardMemoryLimitMb > 0) + .WithMessage("The soft memory limit has to be at or below the hard limit, " + + "or it can never be reached."); } } } diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj index fc19b50c..f3228939 100644 --- a/SW.Bitween.Api/SW.Bitween.Api.csproj +++ b/SW.Bitween.Api/SW.Bitween.Api.csproj @@ -6,7 +6,7 @@ - + diff --git a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs index 1d7a2813..daacad84 100644 --- a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs +++ b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs @@ -147,7 +147,14 @@ await _adapters.StartExclusiveAsync(new AdapterSpec // The instance key IS the data source id, which is how the sink knows which // gateway an inbound message belongs to. InstanceKey = dataSource.Id.ToString(), - StartupValues = startupValues + StartupValues = startupValues, + + // Zero leaves the host's own default in place; the spec only overrides when + // an operator has actually chosen a ceiling for this connection. + SoftMemoryLimitBytes = Megabytes(dataSource.SoftMemoryLimitMb), + HardMemoryLimitBytes = Megabytes(dataSource.HardMemoryLimitMb), + CpuPercentLimit = dataSource.CpuPercentLimit, + CpuLimitSamples = dataSource.CpuLimitSamples }, cancellationToken); _running[dataSource.Id] = fingerprint; @@ -265,8 +272,18 @@ private static Dictionary BuildStartupValues(DataSource dataSour return values; } + private static long Megabytes(int megabytes) => + megabytes > 0 ? (long)megabytes * 1024 * 1024 : 0; + + /// + /// What the adapter was started WITH. The memory ceilings belong in here as much as the + /// connection settings do: they are applied to the process at launch, so raising one has no + /// effect until the adapter restarts — and the fingerprint is what decides that it should. + /// private static string Fingerprint(DataSource dataSource, Dictionary startupValues) => dataSource.AdapterId + "|" + + dataSource.SoftMemoryLimitMb + "|" + dataSource.HardMemoryLimitMb + "|" + + dataSource.CpuPercentLimit + "|" + dataSource.CpuLimitSamples + "|" + string.Join(";", startupValues.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}")); /// diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index 4ed90dc3..f2794a01 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -31,7 +31,7 @@ - + diff --git a/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs b/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs index 0f2252dc..d37f61bd 100644 --- a/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs @@ -7,7 +7,12 @@ using SW.Bitween.Domain; using SW.Bitween.Domain.DataSources; using SW.Bitween.Domain.Gateway; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Services.Cluster; +using SW.Bitween.Services.DataSources; +using SW.Serverless.Resident; using SW.Bitween.Model; using SW.PrimitiveTypes; using Xunit; @@ -308,6 +313,191 @@ public async Task A_gateway_can_be_moved_from_internal_to_external_and_back() Assert.Null(internalAgain.Endpoint); } + // ---------------------------------------------------------------- memory ceilings + + /// + /// The ceilings have to reach the adapter's process, not just the database. + /// + /// They are applied at launch — a GC heap hard limit on the child process — so the supervisor + /// has to fold them into the fingerprint it compares each pass. Otherwise raising a limit + /// saves cleanly, changes nothing, and the adapter keeps running under the old one with + /// nothing to say it did not take. + /// + [Fact] + public async Task Changing_a_memory_ceiling_restarts_the_adapter() + { + var dataSourceId = await CreateAsync( + new Dictionary(_fixture.ExternalRabbitProperties)); + await CreateGatewayAsync(dataSourceId, Unique("mem")); + + var host = _fixture.App.Services.GetRequiredService(); + + // Disposed with the test: the election holds the exclusive queue that IS the lock, so + // leaking one leaves this data source owned by a node that no longer exists — and every + // later reconcile quietly declines to start it. + using var election = Node(); + var supervisor = Supervisor(host, election); + + try + { + await supervisor.ReconcileAsync(); + + var before = host.Describe() + .FirstOrDefault(h => h.InstanceKey == dataSourceId.ToString())?.ProcessId; + Assert.NotNull(before); + + var row = await GetAsync(dataSourceId); + row.HardMemoryLimitMb = 512; + await UpdateAsync(dataSourceId, row); + + await supervisor.ReconcileAsync(); + + var after = host.Describe() + .FirstOrDefault(h => h.InstanceKey == dataSourceId.ToString())?.ProcessId; + Assert.NotNull(after); + Assert.NotEqual(before, after); + } + finally + { + try { await supervisor.StopAsync(default); } catch { } + try { await host.StopAsync(BusAdapters.RabbitMq, dataSourceId.ToString(), drain: false); } + catch { } + supervisor.Dispose(); + } + } + + /// + /// A soft ceiling above the hard one can never fire: the runtime fails the allocation before + /// the supervisor ever sees the soft breach, so the graceful recycle it was configured for + /// silently never happens. + /// + [Fact] + public async Task A_soft_ceiling_above_the_hard_one_is_refused() + { + var dataSourceId = await CreateAsync(new Dictionary()); + + var row = await GetAsync(dataSourceId); + row.SoftMemoryLimitMb = 900; + row.HardMemoryLimitMb = 256; + + var error = await Assert.ThrowsAnyAsync(() => UpdateAsync(dataSourceId, row)); + Assert.Contains("hard limit", error.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Zero_means_leave_the_host_default_alone() + { + var dataSourceId = await CreateAsync(new Dictionary()); + var row = await GetAsync(dataSourceId); + + Assert.Equal(0, row.SoftMemoryLimitMb); + Assert.Equal(0, row.HardMemoryLimitMb); + } + + /// + /// The CPU ceiling has to reach the adapter's process the same way the memory ones do, which + /// means the supervisor has to fold it into the fingerprint. Otherwise it saves cleanly, + /// changes nothing, and the adapter runs on under the old rule with nothing to say so. + /// + [Fact] + public async Task Changing_the_cpu_ceiling_restarts_the_adapter() + { + var dataSourceId = await CreateAsync( + new Dictionary(_fixture.ExternalRabbitProperties)); + await CreateGatewayAsync(dataSourceId, Unique("cpu")); + + var host = _fixture.App.Services.GetRequiredService(); + + // Disposed with the test: the election holds the exclusive queue that IS the lock, so + // leaking one leaves this data source owned by a node that no longer exists — and every + // later reconcile quietly declines to start it. + using var election = Node(); + var supervisor = Supervisor(host, election); + + try + { + await supervisor.ReconcileAsync(); + + var before = host.Describe() + .FirstOrDefault(h => h.InstanceKey == dataSourceId.ToString())?.ProcessId; + Assert.NotNull(before); + + var row = await GetAsync(dataSourceId); + row.CpuPercentLimit = 40; + row.CpuLimitSamples = 5; + await UpdateAsync(dataSourceId, row); + + await supervisor.ReconcileAsync(); + + var after = host.Describe() + .FirstOrDefault(h => h.InstanceKey == dataSourceId.ToString())?.ProcessId; + Assert.NotNull(after); + Assert.NotEqual(before, after); + } + finally + { + try { await supervisor.StopAsync(default); } catch { } + try { await host.StopAsync(BusAdapters.RabbitMq, dataSourceId.ToString(), drain: false); } + catch { } + supervisor.Dispose(); + } + } + + /// + /// The CPU figure is a share of the whole node, so anything above 100 can never be reached — + /// the ceiling would sit there looking configured and never fire once. + /// + [Fact] + public async Task A_cpu_ceiling_above_one_hundred_percent_is_refused() + { + var dataSourceId = await CreateAsync(new Dictionary()); + + var row = await GetAsync(dataSourceId); + row.CpuPercentLimit = 250; + + var error = await Assert.ThrowsAnyAsync(() => UpdateAsync(dataSourceId, row)); + Assert.Contains("never be reached", error.Message); + } + + // ---------------------------------------------------------------- inspect + + /// + /// Discover and GetStats are relayed; Publish is not. + /// + /// The adapter exposes Publish too, and it writes to the customer's broker. This endpoint is + /// guarded by View, so a passthrough would let a read-level grant publish by naming it in a + /// request body. + /// + [Fact] + public async Task Only_read_only_commands_are_relayed() + { + var dataSourceId = await CreateAsync(new Dictionary()); + + var error = await Assert.ThrowsAnyAsync(() => InspectAsync(dataSourceId, "Publish")); + Assert.Contains("Publish", error.Message); + + // The allowed ones get through to the "is it running here" answer rather than being + // rejected out of hand. + var discover = await InspectAsync(dataSourceId, "Discover"); + Assert.False(discover.Ran); + Assert.Contains("not running", discover.Error, StringComparison.OrdinalIgnoreCase); + } + + /// + /// A data source no node is running answers plainly rather than erroring. A broker connection + /// is exclusive, so "not here" is the normal answer on every node but one. + /// + [Fact] + public async Task Inspecting_a_connection_this_node_does_not_hold_says_so() + { + var dataSourceId = await CreateAsync(new Dictionary()); + + var result = await InspectAsync(dataSourceId, "GetStats"); + + Assert.False(result.Ran); + Assert.NotNull(result.Error); + } + // ---------------------------------------------------------------- helpers private static string Unique(string prefix) => $"{prefix}-{Guid.NewGuid():N}"[..20]; @@ -350,7 +540,11 @@ private async Task UpdateAsync(int id, DataSourceRow row) Properties = row.Properties, SecretProperties = row.SecretProperties, Inactive = row.Inactive, - DeduplicationWindowDays = row.DeduplicationWindowDays + DeduplicationWindowDays = row.DeduplicationWindowDays, + SoftMemoryLimitMb = row.SoftMemoryLimitMb, + HardMemoryLimitMb = row.HardMemoryLimitMb, + CpuPercentLimit = row.CpuPercentLimit, + CpuLimitSamples = row.CpuLimitSamples }); } @@ -379,6 +573,25 @@ private async Task StoredPropertyAsync(int id, string name) return stored.Properties.TryGetValue(name, out var value) ? value : null; } + private RabbitMqLeaderElection Node() => new( + _fixture.App.Services.GetRequiredService(), + _fixture.App.Services, + _fixture.App.Services.GetRequiredService() + .CreateLogger()); + + private BusProviderSupervisor Supervisor(IResidentAdapterHost host, ILeaderElection election) => new( + _fixture.App.Services, host, election, + _fixture.App.Services.GetRequiredService() + .CreateLogger()); + + private async Task InspectAsync(int id, string command) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + return (DataSourceInspectResult)await handler.Handle(id, new DataSourceInspectRequest { Command = command }); + } + private async Task CreateDocumentAsync() { await using var scope = _fixture.CreateScope(); diff --git a/SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.Designer.cs b/SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.Designer.cs new file mode 100644 index 00000000..e0eadc6f --- /dev/null +++ b/SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.Designer.cs @@ -0,0 +1,2296 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using SW.Bitween.MsSql; + +#nullable disable + +namespace SW.Bitween.MsSql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906132336_AdapterMemoryLimits")] + partial class AdapterMemoryLimits + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.HasSequence("DocumentIds"); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Deleted") + .HasColumnType("bit"); + + b.Property("Disabled") + .HasColumnType("bit"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("EmailProvider") + .HasColumnType("tinyint"); + + b.Property("FailedLoginCount") + .HasColumnType("int"); + + b.Property("LockoutEnd") + .HasColumnType("datetime2"); + + b.Property("LoginMethods") + .HasColumnType("tinyint"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Role") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique() + .HasFilter("[Email] IS NOT NULL"); + + b.ToTable("Accounts", (string)null); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("RoleId") + .HasColumnType("int"); + + b.HasKey("AccountId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AccountRoles", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("LoginMethod") + .HasColumnType("tinyint"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.ToTable("RefreshTokens", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("IsSystem") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Permissions") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles", (string)null); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime2"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("HardMemoryLimitMb") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime2"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("SecretProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftMemoryLimitMb") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime2"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("On"); + + b.ToTable("DelayedRetries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValueSql("NEXT VALUE FOR [DocumentIds]"); + + SqlServerPropertyBuilderExtensions.UseSequence(b.Property("Id"), "DocumentIds"); + + b.Property("BusEnabled") + .HasColumnType("bit"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Code") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("bit"); + + b.Property("DocumentFormat") + .HasColumnType("int"); + + b.Property("DuplicateInterval") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("PromotedProperties") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("BusMessageTypeName") + .IsUnique() + .HasFilter("[BusMessageTypeName] IS NOT NULL"); + + b.HasIndex("Code") + .IsUnique() + .HasFilter("[Code] IS NOT NULL"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Documents", (string)null); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("StateAfter") + .HasColumnType("nvarchar(max)"); + + b.Property("StateBefore") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("DocumentId"); + + b.ToTable("DocumentTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("UrlName") + .IsUnique(); + + b.ToTable("ApiGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("int"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("ApiGatewayPartners", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("DocumentId"); + + b.ToTable("BusGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("MatchExpression") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("BusGatewayId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("BusGatewayRoutes", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Values") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("GlobalAdapterValuesSets", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("RunOnBadResult") + .HasColumnType("bit"); + + b.Property("RunOnFailedResult") + .HasColumnType("bit"); + + b.Property("RunOnSubscriptions") + .HasColumnType("nvarchar(max)"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.ToTable("Notifiers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("bit"); + + b.Property("Data") + .HasColumnType("nvarchar(max)"); + + b.Property("FileName") + .HasColumnType("nvarchar(max)"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("OnHoldXchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("Partners", (string)null); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ExchangeIds") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("Outcome") + .HasColumnType("int"); + + b.Property("StartedOn") + .HasColumnType("datetime2"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId", "StartedOn"); + + b.ToTable("ReceiveAttempts", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("AlertMode") + .HasColumnType("tinyint"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryAlertOverrides", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("AttemptsUsed") + .HasColumnType("int"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("datetime2"); + + b.Property("LastAttemptOn") + .HasColumnType("datetime2"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryGroupUsages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Groups") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.ToTable("RetryPolicies", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Value") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Settings", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("datetime2"); + + b.Property("AggregationForId") + .HasColumnType("int"); + + b.Property("AggregationTarget") + .HasColumnType("tinyint"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CustomRetryPolicy") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentFilter") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("IsRunning") + .HasColumnType("bit"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("MatchExpression") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("PausedOn") + .HasColumnType("datetime2"); + + b.Property("ReceiveOn") + .HasColumnType("datetime2"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ReceiverProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryPolicyId") + .HasColumnType("int"); + + b.Property("Temporary") + .HasColumnType("bit"); + + b.Property("Type") + .HasColumnType("tinyint"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ValidatorProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("WorkGroupId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AggregationForId"); + + b.HasIndex("CategoryId"); + + b.HasIndex("DocumentId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("ResponseSubscriptionId"); + + b.HasIndex("RetryPolicyId"); + + b.HasIndex("WorkGroupId"); + + b.ToTable("Subscriptions", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("nvarchar(450)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique() + .HasFilter("[Code] IS NOT NULL"); + + b.ToTable("SubscriptionCategory"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("StateAfter") + .HasColumnType("nvarchar(max)"); + + b.Property("StateBefore") + .HasColumnType("nvarchar(max)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("SubscriptionTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Options") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("CorrelationId") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("InputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("InputSize") + .HasColumnType("int"); + + b.Property("ManualRetry") + .HasColumnType("bit"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryFor") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("StartedOn") + .HasColumnType("datetime2"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId"); + + b.HasIndex("InputHash"); + + b.HasIndex("RetryFor"); + + b.HasIndex("StartedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("Xchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AggregatedOn") + .HasColumnType("datetime2"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("AggregationXchangeId"); + + b.ToTable("XchangeAggregations", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DeliveredOn") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeliveredOn"); + + b.ToTable("XchangeDeliveries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("NotifierId") + .HasColumnType("int"); + + b.Property("NotifierName") + .HasColumnType("nvarchar(max)"); + + b.Property("Success") + .HasColumnType("bit"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.ToTable("XchangeNotifications", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("Hits") + .HasMaxLength(2000) + .IsUnicode(false) + .HasColumnType("varchar(2000)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("PropertiesRaw") + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("PropertiesRaw"); + + b.ToTable("XchangePromotedProperties", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AttemptNumber") + .HasColumnType("int"); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("OutputBad") + .HasColumnType("bit"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("OutputHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OutputSize") + .HasColumnType("int"); + + b.Property("ResponseBad") + .HasColumnType("bit"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ResponseSize") + .HasColumnType("int"); + + b.Property("ResponseXchangeId") + .HasColumnType("nvarchar(max)"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("RetryGroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("Success") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.HasIndex("RetryGroupId"); + + b.ToTable("XchangeResults", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("bit"); + + b.ToTable((string)null); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("nvarchar(max)") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("datetime2") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("nvarchar(max)") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("datetime2") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("bit") + .HasColumnName("success"); + + b.HasKey("Id"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("varbinary(max)") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_blob_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("nvarchar(450)") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("varbinary(max)") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("QRTZ_calendars", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("nvarchar(450)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_cron_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("nvarchar(450)") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bit") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bit") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_QRTZ_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_QRTZ_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_QRTZ_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_QRTZ_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_QRTZ_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_nm_gp"); + + b.ToTable("QRTZ_fired_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("nvarchar(450)") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bit") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bit") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bit") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("varbinary(max)") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bit") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("QRTZ_job_details", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("nvarchar(450)") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("QRTZ_locks", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("QRTZ_paused_trigger_grps", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("nvarchar(450)") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("QRTZ_scheduler_state", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bit") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bit") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("int") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("int") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("nvarchar(450)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simprop_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simple_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("nvarchar(450)") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("nvarchar(450)") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("varbinary(max)") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("int") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("QRTZ_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b1.HasKey("PartnerId", "Id"); + + b1.HasIndex("Key") + .IsUnique(); + + b1.ToTable("PartnerApiCredentials", (string)null); + + b1.WithOwner() + .HasForeignKey("PartnerId"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_AggFor"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_RespSub"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("bit"); + + b1.Property("On") + .HasColumnType("bigint"); + + b1.Property("Recurrence") + .HasColumnType("tinyint"); + + b1.HasKey("SubscriptionId", "Id"); + + b1.ToTable("SubscriptionSchedules", (string)null); + + b1.WithOwner() + .HasForeignKey("SubscriptionId"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.cs b/SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.cs new file mode 100644 index 00000000..6b038250 --- /dev/null +++ b/SW.Bitween.MsSql/Migrations/20260906132336_AdapterMemoryLimits.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.MsSql.Migrations +{ + /// + public partial class AdapterMemoryLimits : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "HardMemoryLimitMb", + table: "DataSources", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "SoftMemoryLimitMb", + table: "DataSources", + type: "int", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "HardMemoryLimitMb", + table: "DataSources"); + + migrationBuilder.DropColumn( + name: "SoftMemoryLimitMb", + table: "DataSources"); + } + } +} diff --git a/SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.Designer.cs b/SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.Designer.cs new file mode 100644 index 00000000..a715ba0a --- /dev/null +++ b/SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.Designer.cs @@ -0,0 +1,2302 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using SW.Bitween.MsSql; + +#nullable disable + +namespace SW.Bitween.MsSql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906141722_AdapterCpuLimits")] + partial class AdapterCpuLimits + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.HasSequence("DocumentIds"); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Deleted") + .HasColumnType("bit"); + + b.Property("Disabled") + .HasColumnType("bit"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("EmailProvider") + .HasColumnType("tinyint"); + + b.Property("FailedLoginCount") + .HasColumnType("int"); + + b.Property("LockoutEnd") + .HasColumnType("datetime2"); + + b.Property("LoginMethods") + .HasColumnType("tinyint"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Role") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique() + .HasFilter("[Email] IS NOT NULL"); + + b.ToTable("Accounts", (string)null); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("RoleId") + .HasColumnType("int"); + + b.HasKey("AccountId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AccountRoles", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("LoginMethod") + .HasColumnType("tinyint"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.ToTable("RefreshTokens", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("IsSystem") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Permissions") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles", (string)null); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime2"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CpuLimitSamples") + .HasColumnType("int"); + + b.Property("CpuPercentLimit") + .HasColumnType("float"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("HardMemoryLimitMb") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime2"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("SecretProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftMemoryLimitMb") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime2"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("On"); + + b.ToTable("DelayedRetries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValueSql("NEXT VALUE FOR [DocumentIds]"); + + SqlServerPropertyBuilderExtensions.UseSequence(b.Property("Id"), "DocumentIds"); + + b.Property("BusEnabled") + .HasColumnType("bit"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Code") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("bit"); + + b.Property("DocumentFormat") + .HasColumnType("int"); + + b.Property("DuplicateInterval") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("PromotedProperties") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("BusMessageTypeName") + .IsUnique() + .HasFilter("[BusMessageTypeName] IS NOT NULL"); + + b.HasIndex("Code") + .IsUnique() + .HasFilter("[Code] IS NOT NULL"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Documents", (string)null); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("StateAfter") + .HasColumnType("nvarchar(max)"); + + b.Property("StateBefore") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("DocumentId"); + + b.ToTable("DocumentTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("UrlName") + .IsUnique(); + + b.ToTable("ApiGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("int"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("ApiGatewayPartners", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("DocumentId"); + + b.ToTable("BusGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("MatchExpression") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("BusGatewayId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("BusGatewayRoutes", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Values") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("GlobalAdapterValuesSets", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("RunOnBadResult") + .HasColumnType("bit"); + + b.Property("RunOnFailedResult") + .HasColumnType("bit"); + + b.Property("RunOnSubscriptions") + .HasColumnType("nvarchar(max)"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.ToTable("Notifiers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("bit"); + + b.Property("Data") + .HasColumnType("nvarchar(max)"); + + b.Property("FileName") + .HasColumnType("nvarchar(max)"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("OnHoldXchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdapterProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("Partners", (string)null); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ExchangeIds") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("Outcome") + .HasColumnType("int"); + + b.Property("StartedOn") + .HasColumnType("datetime2"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId", "StartedOn"); + + b.ToTable("ReceiveAttempts", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("AlertMode") + .HasColumnType("tinyint"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryAlertOverrides", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("AttemptsUsed") + .HasColumnType("int"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("datetime2"); + + b.Property("LastAttemptOn") + .HasColumnType("datetime2"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryGroupUsages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Groups") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.ToTable("RetryPolicies", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.Property("Value") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Settings", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("datetime2"); + + b.Property("AggregationForId") + .HasColumnType("int"); + + b.Property("AggregationTarget") + .HasColumnType("tinyint"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CustomRetryPolicy") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentFilter") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("Inactive") + .HasColumnType("bit"); + + b.Property("IsRunning") + .HasColumnType("bit"); + + b.Property("LastException") + .HasColumnType("nvarchar(max)"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("MatchExpression") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("PausedOn") + .HasColumnType("datetime2"); + + b.Property("ReceiveOn") + .HasColumnType("datetime2"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ReceiverProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryPolicyId") + .HasColumnType("int"); + + b.Property("Temporary") + .HasColumnType("bit"); + + b.Property("Type") + .HasColumnType("tinyint"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ValidatorProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("WorkGroupId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AggregationForId"); + + b.HasIndex("CategoryId"); + + b.HasIndex("DocumentId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("ResponseSubscriptionId"); + + b.HasIndex("RetryPolicyId"); + + b.HasIndex("WorkGroupId"); + + b.ToTable("Subscriptions", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("nvarchar(450)"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedOn") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique() + .HasFilter("[Code] IS NOT NULL"); + + b.ToTable("SubscriptionCategory"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("StateAfter") + .HasColumnType("nvarchar(max)"); + + b.Property("StateBefore") + .HasColumnType("nvarchar(max)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("SubscriptionTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Options") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("CorrelationId") + .HasColumnType("nvarchar(max)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("InputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("InputSize") + .HasColumnType("int"); + + b.Property("ManualRetry") + .HasColumnType("bit"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("nvarchar(max)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryFor") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("StartedOn") + .HasColumnType("datetime2"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId"); + + b.HasIndex("InputHash"); + + b.HasIndex("RetryFor"); + + b.HasIndex("StartedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("Xchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AggregatedOn") + .HasColumnType("datetime2"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("AggregationXchangeId"); + + b.ToTable("XchangeAggregations", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DeliveredOn") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeliveredOn"); + + b.ToTable("XchangeDeliveries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("NotifierId") + .HasColumnType("int"); + + b.Property("NotifierName") + .HasColumnType("nvarchar(max)"); + + b.Property("Success") + .HasColumnType("bit"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.ToTable("XchangeNotifications", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("Hits") + .HasMaxLength(2000) + .IsUnicode(false) + .HasColumnType("varchar(2000)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("PropertiesRaw") + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("PropertiesRaw"); + + b.ToTable("XchangePromotedProperties", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AttemptNumber") + .HasColumnType("int"); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishedOn") + .HasColumnType("datetime2"); + + b.Property("OutputBad") + .HasColumnType("bit"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("OutputHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OutputSize") + .HasColumnType("int"); + + b.Property("ResponseBad") + .HasColumnType("bit"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ResponseSize") + .HasColumnType("int"); + + b.Property("ResponseXchangeId") + .HasColumnType("nvarchar(max)"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("RetryGroupId") + .HasColumnType("uniqueidentifier"); + + b.Property("Success") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.HasIndex("RetryGroupId"); + + b.ToTable("XchangeResults", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("bit"); + + b.ToTable((string)null); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("nvarchar(max)") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("datetime2") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("nvarchar(max)") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("datetime2") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("bit") + .HasColumnName("success"); + + b.HasKey("Id"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("varbinary(max)") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_blob_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("nvarchar(450)") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("varbinary(max)") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("QRTZ_calendars", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("nvarchar(450)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_cron_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("nvarchar(450)") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bit") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bit") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_QRTZ_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_QRTZ_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_QRTZ_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_QRTZ_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_QRTZ_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_nm_gp"); + + b.ToTable("QRTZ_fired_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("nvarchar(450)") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bit") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bit") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bit") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("varbinary(max)") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bit") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("QRTZ_job_details", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("nvarchar(450)") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("QRTZ_locks", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("QRTZ_paused_trigger_grps", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("nvarchar(450)") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("QRTZ_scheduler_state", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bit") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bit") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("int") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("int") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("nvarchar(450)") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("nvarchar(450)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simprop_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simple_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("nvarchar(450)") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("nvarchar(450)") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("varbinary(max)") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("int") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("nvarchar(450)") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("QRTZ_triggers", "dbo"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b1.HasKey("PartnerId", "Id"); + + b1.HasIndex("Key") + .IsUnique(); + + b1.ToTable("PartnerApiCredentials", (string)null); + + b1.WithOwner() + .HasForeignKey("PartnerId"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_AggFor"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_RespSub"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("bit"); + + b1.Property("On") + .HasColumnType("bigint"); + + b1.Property("Recurrence") + .HasColumnType("tinyint"); + + b1.HasKey("SubscriptionId", "Id"); + + b1.ToTable("SubscriptionSchedules", (string)null); + + b1.WithOwner() + .HasForeignKey("SubscriptionId"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.cs b/SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.cs new file mode 100644 index 00000000..6f88f3fa --- /dev/null +++ b/SW.Bitween.MsSql/Migrations/20260906141722_AdapterCpuLimits.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.MsSql.Migrations +{ + /// + public partial class AdapterCpuLimits : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "CpuLimitSamples", + table: "DataSources", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "CpuPercentLimit", + table: "DataSources", + type: "float", + nullable: false, + defaultValue: 0.0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "CpuLimitSamples", + table: "DataSources"); + + migrationBuilder.DropColumn( + name: "CpuPercentLimit", + table: "DataSources"); + } + } +} diff --git a/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs b/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs index dde31f34..04209918 100644 --- a/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs +++ b/SW.Bitween.MsSql/Migrations/BitweenDbContextModelSnapshot.cs @@ -255,6 +255,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("ConsecutiveFailures") .HasColumnType("int"); + b.Property("CpuLimitSamples") + .HasColumnType("int"); + + b.Property("CpuPercentLimit") + .HasColumnType("float"); + b.Property("CreatedBy") .HasColumnType("nvarchar(max)"); @@ -264,6 +270,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("DeduplicationWindowDays") .HasColumnType("int"); + b.Property("HardMemoryLimitMb") + .HasColumnType("int"); + b.Property("Inactive") .HasColumnType("bit"); @@ -303,6 +312,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("SecretProperties") .HasColumnType("nvarchar(max)"); + b.Property("SoftMemoryLimitMb") + .HasColumnType("int"); + b.HasKey("Id"); b.HasIndex("Name") diff --git a/SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.Designer.cs b/SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.Designer.cs new file mode 100644 index 00000000..b91a4fdf --- /dev/null +++ b/SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.Designer.cs @@ -0,0 +1,2289 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using SW.Bitween.MySql; + +#nullable disable + +namespace SW.Bitween.MySql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906132331_AdapterMemoryLimits")] + partial class AdapterMemoryLimits + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + MySqlModelBuilderExtensions.AutoIncrementColumns(modelBuilder); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Deleted") + .HasColumnType("tinyint(1)"); + + b.Property("Disabled") + .HasColumnType("tinyint(1)"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("EmailProvider") + .HasColumnType("tinyint unsigned"); + + b.Property("FailedLoginCount") + .HasColumnType("int"); + + b.Property("LockoutEnd") + .HasColumnType("datetime(6)"); + + b.Property("LoginMethods") + .HasColumnType("tinyint unsigned"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Role") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique(); + + b.ToTable("Accounts", (string)null); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("RoleId") + .HasColumnType("int"); + + b.HasKey("AccountId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AccountRoles", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("LoginMethod") + .HasColumnType("tinyint unsigned"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.ToTable("RefreshTokens", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b.Property("IsSystem") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("Permissions") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles", (string)null); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime(6)"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("HardMemoryLimitMb") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime(6)"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("SecretProperties") + .HasColumnType("longtext"); + + b.Property("SoftMemoryLimitMb") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime(6)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("On") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("On"); + + b.ToTable("DelayedRetries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusEnabled") + .HasColumnType("tinyint(1)"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Code") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("tinyint(1)"); + + b.Property("DocumentFormat") + .HasColumnType("int"); + + b.Property("DuplicateInterval") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("PromotedProperties") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("BusMessageTypeName") + .IsUnique(); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Documents", (string)null); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("StateAfter") + .HasColumnType("longtext"); + + b.Property("StateBefore") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("DocumentId"); + + b.ToTable("DocumentTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("UrlName") + .IsUnique(); + + b.ToTable("ApiGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("int"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("ApiGatewayPartners", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("DocumentId"); + + b.ToTable("BusGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("MatchExpression") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("BusGatewayId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("BusGatewayRoutes", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Name") + .HasColumnType("longtext"); + + b.Property("Values") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("GlobalAdapterValuesSets", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("RunOnBadResult") + .HasColumnType("tinyint(1)"); + + b.Property("RunOnFailedResult") + .HasColumnType("tinyint(1)"); + + b.Property("RunOnSubscriptions") + .HasColumnType("longtext"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.ToTable("Notifiers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("tinyint(1)"); + + b.Property("Data") + .HasColumnType("longtext"); + + b.Property("FileName") + .HasColumnType("longtext"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("varchar(1024)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("OnHoldXchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterProperties") + .HasColumnType("longtext"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("Partners", (string)null); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasMaxLength(4000) + .HasColumnType("varchar(4000)"); + + b.Property("ExchangeIds") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("Outcome") + .HasColumnType("int"); + + b.Property("StartedOn") + .HasColumnType("datetime(6)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId", "StartedOn"); + + b.ToTable("ReceiveAttempts", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("char(36)"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("longtext"); + + b.Property("AlertMode") + .HasColumnType("tinyint unsigned"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryAlertOverrides", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("char(36)"); + + b.Property("AttemptsUsed") + .HasColumnType("int"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("LastAttemptOn") + .HasColumnType("datetime(6)"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryGroupUsages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("longtext"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Groups") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("RetryPolicies", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Value") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("Settings", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("datetime(6)"); + + b.Property("AggregationForId") + .HasColumnType("int"); + + b.Property("AggregationTarget") + .HasColumnType("tinyint unsigned"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CustomRetryPolicy") + .HasColumnType("longtext"); + + b.Property("DocumentFilter") + .HasColumnType("longtext"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("IsRunning") + .HasColumnType("tinyint(1)"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("longtext"); + + b.Property("MatchExpression") + .HasColumnType("longtext"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("PausedOn") + .HasColumnType("datetime(6)"); + + b.Property("ReceiveOn") + .HasColumnType("datetime(6)"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ReceiverProperties") + .HasColumnType("longtext"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryPolicyId") + .HasColumnType("int"); + + b.Property("Temporary") + .HasColumnType("tinyint(1)"); + + b.Property("Type") + .HasColumnType("tinyint unsigned"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ValidatorProperties") + .HasColumnType("longtext"); + + b.Property("WorkGroupId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AggregationForId"); + + b.HasIndex("CategoryId"); + + b.HasIndex("DocumentId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("ResponseSubscriptionId"); + + b.HasIndex("RetryPolicyId"); + + b.HasIndex("WorkGroupId"); + + b.ToTable("Subscriptions", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("varchar(255)"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Description") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("SubscriptionCategory"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("StateAfter") + .HasColumnType("longtext"); + + b.Property("StateBefore") + .HasColumnType("longtext"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("SubscriptionTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("Name") + .HasColumnType("longtext"); + + b.Property("Options") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("CorrelationId") + .HasColumnType("longtext"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("InputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("InputSize") + .HasColumnType("int"); + + b.Property("ManualRetry") + .HasColumnType("tinyint(1)"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("longtext"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("varchar(1024)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryFor") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("StartedOn") + .HasColumnType("datetime(6)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId"); + + b.HasIndex("InputHash"); + + b.HasIndex("RetryFor"); + + b.HasIndex("StartedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("Xchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AggregatedOn") + .HasColumnType("datetime(6)"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("AggregationXchangeId"); + + b.ToTable("XchangeAggregations", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DeliveredOn") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("DeliveredOn"); + + b.ToTable("XchangeDeliveries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("NotifierId") + .HasColumnType("int"); + + b.Property("NotifierName") + .HasColumnType("longtext"); + + b.Property("Success") + .HasColumnType("tinyint(1)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.ToTable("XchangeNotifications", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("Hits") + .HasMaxLength(2000) + .IsUnicode(false) + .HasColumnType("varchar(2000)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("PropertiesRaw") + .HasColumnType("varchar(255)"); + + b.HasKey("Id"); + + b.HasIndex("PropertiesRaw"); + + b.ToTable("XchangePromotedProperties", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AttemptNumber") + .HasColumnType("int"); + + b.Property("Exception") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("OutputBad") + .HasColumnType("tinyint(1)"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("OutputHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OutputSize") + .HasColumnType("int"); + + b.Property("ResponseBad") + .HasColumnType("tinyint(1)"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("ResponseSize") + .HasColumnType("int"); + + b.Property("ResponseXchangeId") + .HasColumnType("longtext"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b.Property("RetryGroupId") + .HasColumnType("char(36)"); + + b.Property("Success") + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.HasIndex("RetryGroupId"); + + b.ToTable("XchangeResults", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("tinyint(1)"); + + b.ToTable((string)null); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("longtext") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("datetime(6)") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("longtext") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("datetime(6)") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("tinyint(1)") + .HasColumnName("success"); + + b.HasKey("Id"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("longblob") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_blob_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("varchar(200)") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("longblob") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("QRTZ_calendars", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("varchar(200)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_cron_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("varchar(200)") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("tinyint(1)") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("tinyint(1)") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_QRTZ_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_QRTZ_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_QRTZ_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_QRTZ_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_QRTZ_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_nm_gp"); + + b.ToTable("QRTZ_fired_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("varchar(200)") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("tinyint(1)") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("tinyint(1)") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("tinyint(1)") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("longblob") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("tinyint(1)") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("QRTZ_job_details", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("varchar(200)") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("QRTZ_locks", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("QRTZ_paused_trigger_grps", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("varchar(200)") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("QRTZ_scheduler_state", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("tinyint(1)") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("tinyint(1)") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("int") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("int") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("varchar(200)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simprop_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simple_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("varchar(200)") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("varchar(200)") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("longblob") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("int") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("QRTZ_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b1.HasKey("PartnerId", "Id"); + + b1.HasIndex("Key") + .IsUnique(); + + b1.ToTable("PartnerApiCredentials", (string)null); + + b1.WithOwner() + .HasForeignKey("PartnerId"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_AggFor"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_RespSub"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("tinyint(1)"); + + b1.Property("On") + .HasColumnType("bigint"); + + b1.Property("Recurrence") + .HasColumnType("tinyint unsigned"); + + b1.HasKey("SubscriptionId", "Id"); + + b1.ToTable("SubscriptionSchedules", (string)null); + + b1.WithOwner() + .HasForeignKey("SubscriptionId"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.cs b/SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.cs new file mode 100644 index 00000000..4e093412 --- /dev/null +++ b/SW.Bitween.MySql/Migrations/20260906132331_AdapterMemoryLimits.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.MySql.Migrations +{ + /// + public partial class AdapterMemoryLimits : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "HardMemoryLimitMb", + table: "DataSources", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "SoftMemoryLimitMb", + table: "DataSources", + type: "int", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "HardMemoryLimitMb", + table: "DataSources"); + + migrationBuilder.DropColumn( + name: "SoftMemoryLimitMb", + table: "DataSources"); + } + } +} diff --git a/SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.Designer.cs b/SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.Designer.cs new file mode 100644 index 00000000..1adbbe35 --- /dev/null +++ b/SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.Designer.cs @@ -0,0 +1,2295 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using SW.Bitween.MySql; + +#nullable disable + +namespace SW.Bitween.MySql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906141717_AdapterCpuLimits")] + partial class AdapterCpuLimits + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + MySqlModelBuilderExtensions.AutoIncrementColumns(modelBuilder); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Deleted") + .HasColumnType("tinyint(1)"); + + b.Property("Disabled") + .HasColumnType("tinyint(1)"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("EmailProvider") + .HasColumnType("tinyint unsigned"); + + b.Property("FailedLoginCount") + .HasColumnType("int"); + + b.Property("LockoutEnd") + .HasColumnType("datetime(6)"); + + b.Property("LoginMethods") + .HasColumnType("tinyint unsigned"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Role") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique(); + + b.ToTable("Accounts", (string)null); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("RoleId") + .HasColumnType("int"); + + b.HasKey("AccountId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AccountRoles", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AccountId") + .HasColumnType("int"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("LoginMethod") + .HasColumnType("tinyint unsigned"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.ToTable("RefreshTokens", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b.Property("IsSystem") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("Permissions") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles", (string)null); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AcquiredOn") + .HasColumnType("datetime(6)"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Term") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("ClusterLeases", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterId") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CpuLimitSamples") + .HasColumnType("int"); + + b.Property("CpuPercentLimit") + .HasColumnType("double"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("int"); + + b.Property("HardMemoryLimitMb") + .HasColumnType("int"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Kind") + .HasColumnType("int"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("LastHeartbeatOn") + .HasColumnType("datetime(6)"); + + b.Property("LastKnownState") + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OwnedByNode") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("SecretProperties") + .HasColumnType("longtext"); + + b.Property("SoftMemoryLimitMb") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DataSources", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .IsUnicode(false) + .HasColumnType("varchar(400)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("SeenOn") + .HasColumnType("datetime(6)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("SeenOn"); + + b.ToTable("InboundMessages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("On") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("On"); + + b.ToTable("DelayedRetries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusEnabled") + .HasColumnType("tinyint(1)"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("Code") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("tinyint(1)"); + + b.Property("DocumentFormat") + .HasColumnType("int"); + + b.Property("DuplicateInterval") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("PromotedProperties") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("BusMessageTypeName") + .IsUnique(); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Documents", (string)null); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("StateAfter") + .HasColumnType("longtext"); + + b.Property("StateBefore") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("DocumentId"); + + b.ToTable("DocumentTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("UrlName") + .IsUnique(); + + b.ToTable("ApiGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("int"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("ApiGatewayPartners", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("DataSourceId") + .HasColumnType("int"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("Endpoint") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("EndpointProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("DataSourceId"); + + b.HasIndex("DocumentId"); + + b.ToTable("BusGateways", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("MatchExpression") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("BusGatewayId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("BusGatewayRoutes", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("Name") + .HasColumnType("longtext"); + + b.Property("Values") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("GlobalAdapterValuesSets", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("RunOnBadResult") + .HasColumnType("tinyint(1)"); + + b.Property("RunOnFailedResult") + .HasColumnType("tinyint(1)"); + + b.Property("RunOnSubscriptions") + .HasColumnType("longtext"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.ToTable("Notifiers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("tinyint(1)"); + + b.Property("Data") + .HasColumnType("longtext"); + + b.Property("FileName") + .HasColumnType("longtext"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("varchar(1024)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("OnHoldXchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AdapterProperties") + .HasColumnType("longtext"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("Partners", (string)null); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasMaxLength(4000) + .HasColumnType("varchar(4000)"); + + b.Property("ExchangeIds") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("Outcome") + .HasColumnType("int"); + + b.Property("StartedOn") + .HasColumnType("datetime(6)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SubscriptionId", "StartedOn"); + + b.ToTable("ReceiveAttempts", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("char(36)"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("longtext"); + + b.Property("AlertMode") + .HasColumnType("tinyint unsigned"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryAlertOverrides", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.Property("GroupId") + .HasColumnType("char(36)"); + + b.Property("AttemptsUsed") + .HasColumnType("int"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("LastAttemptOn") + .HasColumnType("datetime(6)"); + + b.HasKey("SubscriptionId", "GroupId"); + + b.ToTable("RetryGroupUsages", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("AlertHandlerProperties") + .HasColumnType("longtext"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Groups") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.HasKey("Id"); + + b.ToTable("RetryPolicies", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.Property("Value") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("Settings", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("datetime(6)"); + + b.Property("AggregationForId") + .HasColumnType("int"); + + b.Property("AggregationTarget") + .HasColumnType("tinyint unsigned"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("ConsecutiveFailures") + .HasColumnType("int"); + + b.Property("CustomRetryPolicy") + .HasColumnType("longtext"); + + b.Property("DocumentFilter") + .HasColumnType("longtext"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("Inactive") + .HasColumnType("tinyint(1)"); + + b.Property("IsRunning") + .HasColumnType("tinyint(1)"); + + b.Property("LastException") + .HasColumnType("longtext"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("longtext"); + + b.Property("MatchExpression") + .HasColumnType("longtext"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("PausedOn") + .HasColumnType("datetime(6)"); + + b.Property("ReceiveOn") + .HasColumnType("datetime(6)"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ReceiverProperties") + .HasColumnType("longtext"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryPolicyId") + .HasColumnType("int"); + + b.Property("Temporary") + .HasColumnType("tinyint(1)"); + + b.Property("Type") + .HasColumnType("tinyint unsigned"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ValidatorProperties") + .HasColumnType("longtext"); + + b.Property("WorkGroupId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AggregationForId"); + + b.HasIndex("CategoryId"); + + b.HasIndex("DocumentId"); + + b.HasIndex("PartnerId"); + + b.HasIndex("ResponseSubscriptionId"); + + b.HasIndex("RetryPolicyId"); + + b.HasIndex("WorkGroupId"); + + b.ToTable("Subscriptions", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("varchar(255)"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("Description") + .HasColumnType("longtext"); + + b.Property("ModifiedBy") + .HasColumnType("longtext"); + + b.Property("ModifiedOn") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("SubscriptionCategory"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("Code") + .HasColumnType("int"); + + b.Property("CreatedBy") + .HasColumnType("longtext"); + + b.Property("CreatedOn") + .HasColumnType("datetime(6)"); + + b.Property("StateAfter") + .HasColumnType("longtext"); + + b.Property("StateBefore") + .HasColumnType("longtext"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("SubscriptionTrail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("varchar(100)"); + + b.Property("Name") + .HasColumnType("longtext"); + + b.Property("Options") + .HasColumnType("longtext"); + + b.HasKey("Id"); + + b.ToTable("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("CorrelationId") + .HasColumnType("longtext"); + + b.Property("DocumentId") + .HasColumnType("int"); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("HandlerProperties") + .HasColumnType("longtext"); + + b.Property("InputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("InputSize") + .HasColumnType("int"); + + b.Property("ManualRetry") + .HasColumnType("tinyint(1)"); + + b.Property("MapperId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("MapperProperties") + .HasColumnType("longtext"); + + b.Property("PartnerId") + .HasColumnType("int"); + + b.Property("References") + .HasMaxLength(1024) + .HasColumnType("varchar(1024)"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("int"); + + b.Property("RetryFor") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("StartedOn") + .HasColumnType("datetime(6)"); + + b.Property("SubscriptionId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId"); + + b.HasIndex("InputHash"); + + b.HasIndex("RetryFor"); + + b.HasIndex("StartedOn"); + + b.HasIndex("SubscriptionId"); + + b.ToTable("Xchanges", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AggregatedOn") + .HasColumnType("datetime(6)"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("AggregationXchangeId"); + + b.ToTable("XchangeAggregations", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("DeliveredOn") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("DeliveredOn"); + + b.ToTable("XchangeDeliveries", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("NotifierId") + .HasColumnType("int"); + + b.Property("NotifierName") + .HasColumnType("longtext"); + + b.Property("Success") + .HasColumnType("tinyint(1)"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.HasKey("Id"); + + b.ToTable("XchangeNotifications", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("Hits") + .HasMaxLength(2000) + .IsUnicode(false) + .HasColumnType("varchar(2000)"); + + b.Property("Properties") + .HasColumnType("longtext"); + + b.Property("PropertiesRaw") + .HasColumnType("varchar(255)"); + + b.HasKey("Id"); + + b.HasIndex("PropertiesRaw"); + + b.ToTable("XchangePromotedProperties", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("AttemptNumber") + .HasColumnType("int"); + + b.Property("Exception") + .HasColumnType("longtext"); + + b.Property("FinishedOn") + .HasColumnType("datetime(6)"); + + b.Property("OutputBad") + .HasColumnType("tinyint(1)"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("OutputHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("OutputSize") + .HasColumnType("int"); + + b.Property("ResponseBad") + .HasColumnType("tinyint(1)"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("varchar(200)"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("varchar(50)"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("varchar(200)"); + + b.Property("ResponseSize") + .HasColumnType("int"); + + b.Property("ResponseXchangeId") + .HasColumnType("longtext"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b.Property("RetryGroupId") + .HasColumnType("char(36)"); + + b.Property("Success") + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.HasIndex("RetryGroupId"); + + b.ToTable("XchangeResults", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("tinyint(1)"); + + b.ToTable((string)null); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("longtext") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("datetime(6)") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("longtext") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("datetime(6)") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("tinyint(1)") + .HasColumnName("success"); + + b.HasKey("Id"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("longblob") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_blob_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("varchar(200)") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("longblob") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("QRTZ_calendars", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("varchar(200)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_cron_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("varchar(200)") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("tinyint(1)") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("tinyint(1)") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_QRTZ_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_QRTZ_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_QRTZ_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_QRTZ_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_QRTZ_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_QRTZ_ft_trig_nm_gp"); + + b.ToTable("QRTZ_fired_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("varchar(200)") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("tinyint(1)") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("tinyint(1)") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("tinyint(1)") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("longblob") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("tinyint(1)") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("QRTZ_job_details", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("varchar(200)") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("QRTZ_locks", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("QRTZ_paused_trigger_grps", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("varchar(200)") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("QRTZ_scheduler_state", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("tinyint(1)") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("tinyint(1)") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("int") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("int") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("varchar(200)") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("varchar(200)") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simprop_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("QRTZ_simple_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("varchar(200)") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("varchar(200)") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("varchar(200)") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("varchar(200)") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("longblob") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("int") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("varchar(200)") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("QRTZ_triggers", (string)null); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("varchar(500)"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("varchar(500)"); + + b1.HasKey("PartnerId", "Id"); + + b1.HasIndex("Key") + .IsUnique(); + + b1.ToTable("PartnerApiCredentials", (string)null); + + b1.WithOwner() + .HasForeignKey("PartnerId"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_AggFor"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_Subscriptions_RespSub"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("int"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("tinyint(1)"); + + b1.Property("On") + .HasColumnType("bigint"); + + b1.Property("Recurrence") + .HasColumnType("tinyint unsigned"); + + b1.HasKey("SubscriptionId", "Id"); + + b1.ToTable("SubscriptionSchedules", (string)null); + + b1.WithOwner() + .HasForeignKey("SubscriptionId"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.cs b/SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.cs new file mode 100644 index 00000000..aba26b28 --- /dev/null +++ b/SW.Bitween.MySql/Migrations/20260906141717_AdapterCpuLimits.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.MySql.Migrations +{ + /// + public partial class AdapterCpuLimits : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "CpuLimitSamples", + table: "DataSources", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "CpuPercentLimit", + table: "DataSources", + type: "double", + nullable: false, + defaultValue: 0.0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "CpuLimitSamples", + table: "DataSources"); + + migrationBuilder.DropColumn( + name: "CpuPercentLimit", + table: "DataSources"); + } + } +} diff --git a/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs b/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs index bc85f746..1186432e 100644 --- a/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs +++ b/SW.Bitween.MySql/Migrations/BitweenDbContextModelSnapshot.cs @@ -252,6 +252,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("ConsecutiveFailures") .HasColumnType("int"); + b.Property("CpuLimitSamples") + .HasColumnType("int"); + + b.Property("CpuPercentLimit") + .HasColumnType("double"); + b.Property("CreatedBy") .HasColumnType("longtext"); @@ -261,6 +267,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("DeduplicationWindowDays") .HasColumnType("int"); + b.Property("HardMemoryLimitMb") + .HasColumnType("int"); + b.Property("Inactive") .HasColumnType("tinyint(1)"); @@ -300,6 +309,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("SecretProperties") .HasColumnType("longtext"); + b.Property("SoftMemoryLimitMb") + .HasColumnType("int"); + b.HasKey("Id"); b.HasIndex("Name") diff --git a/SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.Designer.cs b/SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.Designer.cs new file mode 100644 index 00000000..2f65bf4a --- /dev/null +++ b/SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.Designer.cs @@ -0,0 +1,2633 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SW.Bitween.Model; +using SW.Bitween.PgSql; + +#nullable disable + +namespace SW.Bitween.PgSql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906132326_AdapterMemoryLimits")] + partial class AdapterMemoryLimits + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("infolink") + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "hstore"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Deleted") + .HasColumnType("boolean") + .HasColumnName("deleted"); + + b.Property("Disabled") + .HasColumnType("boolean") + .HasColumnName("disabled"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("display_name"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("email"); + + b.Property("EmailProvider") + .HasColumnType("smallint") + .HasColumnName("email_provider"); + + b.Property("FailedLoginCount") + .HasColumnType("integer") + .HasColumnName("failed_login_count"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone") + .HasColumnName("lockout_end"); + + b.Property("LoginMethods") + .HasColumnType("smallint") + .HasColumnName("login_methods"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("character varying(500)") + .HasColumnName("password"); + + b.Property("Role") + .HasColumnType("integer") + .HasColumnName("role"); + + b.HasKey("Id") + .HasName("pk_accounts"); + + b.HasIndex("Email") + .IsUnique() + .HasDatabaseName("ix_accounts_email"); + + b.ToTable("Accounts", "infolink"); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("integer") + .HasColumnName("account_id"); + + b.Property("RoleId") + .HasColumnType("integer") + .HasColumnName("role_id"); + + b.HasKey("AccountId", "RoleId") + .HasName("pk_account_roles"); + + b.HasIndex("RoleId") + .HasDatabaseName("ix_account_roles_role_id"); + + b.ToTable("AccountRoles", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("integer") + .HasColumnName("account_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("LoginMethod") + .HasColumnType("smallint") + .HasColumnName("login_method"); + + b.HasKey("Id") + .HasName("pk_refresh_tokens"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_refresh_tokens_account_id"); + + b.ToTable("RefreshTokens", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("description"); + + b.Property("IsSystem") + .HasColumnType("boolean") + .HasColumnName("is_system"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("Permissions") + .HasColumnType("text") + .HasColumnName("permissions"); + + b.HasKey("Id") + .HasName("pk_roles"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_roles_name"); + + b.ToTable("Roles", "infolink"); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("id"); + + b.Property("AcquiredOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("acquired_on"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("owner_node"); + + b.Property("Term") + .HasColumnType("bigint") + .HasColumnName("term"); + + b.HasKey("Id") + .HasName("pk_cluster_lease"); + + b.ToTable("cluster_lease", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdapterId") + .HasColumnType("text") + .HasColumnName("adapter_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("integer") + .HasColumnName("deduplication_window_days"); + + b.Property("HardMemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("hard_memory_limit_mb"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Kind") + .HasColumnType("integer") + .HasColumnName("kind"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("LastHeartbeatOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_heartbeat_on"); + + b.Property("LastKnownState") + .HasColumnType("text") + .HasColumnName("last_known_state"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("OwnedByNode") + .HasColumnType("text") + .HasColumnName("owned_by_node"); + + b.Property>("Properties") + .HasColumnType("hstore") + .HasColumnName("properties"); + + b.PrimitiveCollection>("SecretProperties") + .HasColumnType("text[]") + .HasColumnName("secret_properties"); + + b.Property("SoftMemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("soft_memory_limit_mb"); + + b.HasKey("Id") + .HasName("pk_data_source"); + + b.ToTable("data_source", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("id"); + + b.Property("DataSourceId") + .HasColumnType("integer") + .HasColumnName("data_source_id"); + + b.Property("SeenOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("seen_on"); + + b.Property("XchangeId") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("xchange_id"); + + b.HasKey("Id") + .HasName("pk_inbound_message"); + + b.HasIndex("DataSourceId") + .HasDatabaseName("ix_inbound_message_data_source_id"); + + b.HasIndex("SeenOn") + .HasDatabaseName("ix_inbound_message_seen_on"); + + b.ToTable("inbound_message", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("On") + .HasColumnType("timestamp with time zone") + .HasColumnName("on"); + + b.HasKey("Id") + .HasName("pk_delayed_retry"); + + b.HasIndex("On") + .HasDatabaseName("ix_delayed_retry_on"); + + b.ToTable("delayed_retry", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusEnabled") + .HasColumnType("boolean") + .HasColumnName("bus_enabled"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("bus_message_type_name"); + + b.Property("Code") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("code"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("boolean") + .HasColumnName("disregards_unfiltered_messages"); + + b.Property("DocumentFormat") + .HasColumnType("integer") + .HasColumnName("document_format"); + + b.Property("DuplicateInterval") + .HasColumnType("integer") + .HasColumnName("duplicate_interval"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("PromotedProperties") + .HasColumnType("jsonb") + .HasColumnName("promoted_properties"); + + b.HasKey("Id") + .HasName("pk_document"); + + b.HasIndex("BusMessageTypeName") + .IsUnique() + .HasDatabaseName("ix_document_bus_message_type_name"); + + b.HasIndex("Code") + .IsUnique() + .HasDatabaseName("ix_document_code"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_document_name"); + + b.ToTable("document", "infolink"); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("Code") + .HasColumnType("integer") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("StateAfter") + .HasColumnType("text") + .HasColumnName("state_after"); + + b.Property("StateBefore") + .HasColumnType("text") + .HasColumnName("state_before"); + + b.HasKey("Id") + .HasName("pk_document_trail"); + + b.HasIndex("CreatedOn") + .HasDatabaseName("ix_document_trail_created_on"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_document_trail_document_id"); + + b.ToTable("document_trail", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("url_name"); + + b.HasKey("Id") + .HasName("pk_api_gateway"); + + b.HasIndex("UrlName") + .IsUnique() + .HasDatabaseName("ix_api_gateway_url_name"); + + b.ToTable("api_gateway", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("integer") + .HasColumnName("api_gateway_id"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId") + .HasName("pk_api_gateway_partner"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_api_gateway_partner_partner_id"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_api_gateway_partner_subscription_id"); + + b.ToTable("api_gateway_partner", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DataSourceId") + .HasColumnType("integer") + .HasColumnName("data_source_id"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property>("EndpointProperties") + .HasColumnType("hstore") + .HasColumnName("endpoint_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_bus_gateway"); + + b.HasIndex("DataSourceId") + .HasDatabaseName("ix_bus_gateway_data_source_id"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_bus_gateway_document_id"); + + b.ToTable("bus_gateway", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("integer") + .HasColumnName("bus_gateway_id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("MatchExpression") + .HasColumnType("text") + .HasColumnName("match_expression"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_bus_gateway_route"); + + b.HasIndex("BusGatewayId") + .HasDatabaseName("ix_bus_gateway_route_bus_gateway_id"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_bus_gateway_route_partner_id"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_bus_gateway_route_subscription_id"); + + b.ToTable("bus_gateway_route", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property>("Values") + .HasColumnType("jsonb") + .HasColumnName("values"); + + b.HasKey("Id") + .HasName("pk_global_adapter_values_set"); + + b.ToTable("global_adapter_values_set", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property("HandlerProperties") + .HasColumnType("text") + .HasColumnName("handler_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("RunOnBadResult") + .HasColumnType("boolean") + .HasColumnName("run_on_bad_result"); + + b.Property("RunOnFailedResult") + .HasColumnType("boolean") + .HasColumnName("run_on_failed_result"); + + b.PrimitiveCollection("RunOnSubscriptions") + .HasColumnType("integer[]") + .HasColumnName("run_on_subscriptions"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("boolean") + .HasColumnName("run_on_successful_result"); + + b.HasKey("Id") + .HasName("pk_notifier"); + + b.ToTable("notifier", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("boolean") + .HasColumnName("bad_data"); + + b.Property("Data") + .HasColumnType("text") + .HasColumnName("data"); + + b.Property("FileName") + .HasColumnType("text") + .HasColumnName("file_name"); + + b.PrimitiveCollection("References") + .HasColumnType("text[]") + .HasColumnName("references"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_on_hold_xchange"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_on_hold_xchange_subscription_id"); + + b.ToTable("on_hold_xchange", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property>("AdapterProperties") + .HasColumnType("jsonb") + .HasColumnName("adapter_properties"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_partner"); + + b.ToTable("partner", "infolink"); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasColumnType("text") + .HasColumnName("error_message"); + + b.PrimitiveCollection("ExchangeIds") + .HasColumnType("text[]") + .HasColumnName("exchange_ids"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("Outcome") + .HasColumnType("integer") + .HasColumnName("outcome"); + + b.Property("StartedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_on"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_receive_attempt"); + + b.HasIndex("SubscriptionId", "StartedOn") + .HasDatabaseName("ix_receive_attempt_subscription_id_started_on"); + + b.ToTable("receive_attempt", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("GroupId") + .HasColumnType("uuid") + .HasColumnName("group_id"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("alert_handler_id"); + + b.Property("AlertHandlerProperties") + .HasColumnType("text") + .HasColumnName("alert_handler_properties"); + + b.Property("AlertMode") + .HasColumnType("smallint") + .HasColumnName("alert_mode"); + + b.HasKey("SubscriptionId", "GroupId") + .HasName("pk_retry_alert_override"); + + b.ToTable("retry_alert_override", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("GroupId") + .HasColumnType("uuid") + .HasColumnName("group_id"); + + b.Property("AttemptsUsed") + .HasColumnType("integer") + .HasColumnName("attempts_used"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("exhausted_notified_on"); + + b.Property("LastAttemptOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempt_on"); + + b.HasKey("SubscriptionId", "GroupId") + .HasName("pk_retry_group_usage"); + + b.ToTable("retry_group_usage", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("alert_handler_id"); + + b.Property("AlertHandlerProperties") + .HasColumnType("text") + .HasColumnName("alert_handler_properties"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Groups") + .HasColumnType("text") + .HasColumnName("groups"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_retry_policy"); + + b.ToTable("retry_policy", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Value") + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Id") + .HasName("pk_settings"); + + b.ToTable("Settings", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("aggregate_on"); + + b.Property("AggregationForId") + .HasColumnType("integer") + .HasColumnName("aggregation_for_id"); + + b.Property("AggregationTarget") + .HasColumnType("smallint") + .HasColumnName("aggregation_target"); + + b.Property("CategoryId") + .HasColumnType("integer") + .HasColumnName("category_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CustomRetryPolicy") + .HasColumnType("text") + .HasColumnName("custom_retry_policy"); + + b.Property>("DocumentFilter") + .HasColumnType("jsonb") + .HasColumnName("document_filter"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("HandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property>("HandlerProperties") + .HasColumnType("jsonb") + .HasColumnName("handler_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("IsRunning") + .HasColumnType("boolean") + .HasColumnName("is_running"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("MapperId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("mapper_id"); + + b.Property>("MapperProperties") + .HasColumnType("jsonb") + .HasColumnName("mapper_properties"); + + b.Property("MatchExpression") + .HasColumnType("text") + .HasColumnName("match_expression"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("PausedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("paused_on"); + + b.Property("ReceiveOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("receive_on"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("receiver_id"); + + b.Property>("ReceiverProperties") + .HasColumnType("jsonb") + .HasColumnName("receiver_properties"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("response_message_type_name"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("integer") + .HasColumnName("response_subscription_id"); + + b.Property("RetryPolicyId") + .HasColumnType("integer") + .HasColumnName("retry_policy_id"); + + b.Property("Temporary") + .HasColumnType("boolean") + .HasColumnName("temporary"); + + b.Property("Type") + .HasColumnType("smallint") + .HasColumnName("type"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("validator_id"); + + b.Property>("ValidatorProperties") + .HasColumnType("jsonb") + .HasColumnName("validator_properties"); + + b.Property("WorkGroupId") + .HasColumnType("integer") + .HasColumnName("work_group_id"); + + b.HasKey("Id") + .HasName("pk_subscription"); + + b.HasIndex("AggregationForId") + .HasDatabaseName("ix_subscription_aggregation_for_id"); + + b.HasIndex("CategoryId") + .HasDatabaseName("ix_subscription_category_id"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_subscription_document_id"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_subscription_partner_id"); + + b.HasIndex("ResponseSubscriptionId") + .HasDatabaseName("ix_subscription_response_subscription_id"); + + b.HasIndex("RetryPolicyId") + .HasDatabaseName("ix_subscription_retry_policy_id"); + + b.HasIndex("WorkGroupId") + .HasDatabaseName("ix_subscription_work_group_id"); + + b.ToTable("subscription", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("text") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.HasKey("Id") + .HasName("pk_subscription_category"); + + b.HasIndex("Code") + .IsUnique() + .HasDatabaseName("ix_subscription_category_code"); + + b.ToTable("subscription_category", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("Code") + .HasColumnType("integer") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("StateAfter") + .HasColumnType("text") + .HasColumnName("state_after"); + + b.Property("StateBefore") + .HasColumnType("text") + .HasColumnName("state_before"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_subscription_trail"); + + b.HasIndex("CreatedOn") + .HasDatabaseName("ix_subscription_trail_created_on"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_subscription_trail_subscription_id"); + + b.ToTable("subscription_trail", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("character varying(100)") + .HasColumnName("bus_message_name"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Options") + .HasColumnType("jsonb") + .HasColumnName("options"); + + b.HasKey("Id") + .HasName("pk_work_group"); + + b.ToTable("work_group", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("CorrelationId") + .HasColumnType("text") + .HasColumnName("correlation_id"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("HandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property>("HandlerProperties") + .HasColumnType("jsonb") + .HasColumnName("handler_properties"); + + b.Property("InputContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("input_content_type"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("input_hash"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("input_name"); + + b.Property("InputSize") + .HasColumnType("integer") + .HasColumnName("input_size"); + + b.Property("ManualRetry") + .HasColumnType("boolean") + .HasColumnName("manual_retry"); + + b.Property("MapperId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("mapper_id"); + + b.Property>("MapperProperties") + .HasColumnType("jsonb") + .HasColumnName("mapper_properties"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.PrimitiveCollection("References") + .HasColumnType("text[]") + .HasColumnName("references"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("response_message_type_name"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("integer") + .HasColumnName("response_subscription_id"); + + b.Property("RetryFor") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("retry_for"); + + b.Property("StartedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_on"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_xchange"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_xchange_document_id"); + + b.HasIndex("InputHash") + .HasDatabaseName("ix_xchange_input_hash"); + + b.HasIndex("RetryFor") + .HasDatabaseName("ix_xchange_retry_for"); + + b.HasIndex("StartedOn") + .HasDatabaseName("ix_xchange_started_on"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_xchange_subscription_id"); + + b.ToTable("xchange", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AggregatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("aggregated_on"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("aggregation_xchange_id"); + + b.HasKey("Id") + .HasName("pk_xchange_aggregation"); + + b.HasIndex("AggregationXchangeId") + .HasDatabaseName("ix_xchange_aggregation_aggregation_xchange_id"); + + b.ToTable("xchange_aggregation", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("DeliveredOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("delivered_on"); + + b.HasKey("Id") + .HasName("pk_xchange_delivery"); + + b.HasIndex("DeliveredOn") + .HasDatabaseName("ix_xchange_delivery_delivered_on"); + + b.ToTable("xchange_delivery", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("text") + .HasColumnName("exception"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("NotifierId") + .HasColumnType("integer") + .HasColumnName("notifier_id"); + + b.Property("NotifierName") + .HasColumnType("text") + .HasColumnName("notifier_name"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("character varying(50)") + .HasColumnName("xchange_id"); + + b.HasKey("Id") + .HasName("pk_xchange_notification"); + + b.ToTable("xchange_notification", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.PrimitiveCollection("Hits") + .HasColumnType("integer[]") + .HasColumnName("hits"); + + b.Property>("Properties") + .HasColumnType("jsonb") + .HasColumnName("properties"); + + b.Property("PropertiesRaw") + .HasColumnType("text") + .HasColumnName("properties_raw"); + + b.HasKey("Id") + .HasName("pk_xchange_promoted_properties"); + + b.HasIndex("PropertiesRaw") + .HasDatabaseName("ix_xchange_promoted_properties_properties_raw"); + + b.ToTable("xchange_promoted_properties", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AttemptNumber") + .HasColumnType("integer") + .HasColumnName("attempt_number"); + + b.Property("Exception") + .HasColumnType("text") + .HasColumnName("exception"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("OutputBad") + .HasColumnType("boolean") + .HasColumnName("output_bad"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("output_content_type"); + + b.Property("OutputHash") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("output_hash"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("output_name"); + + b.Property("OutputSize") + .HasColumnType("integer") + .HasColumnName("output_size"); + + b.Property("ResponseBad") + .HasColumnType("boolean") + .HasColumnName("response_bad"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("response_content_type"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("response_hash"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("response_name"); + + b.Property("ResponseSize") + .HasColumnType("integer") + .HasColumnName("response_size"); + + b.Property("ResponseXchangeId") + .HasColumnType("text") + .HasColumnName("response_xchange_id"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("retry_blocked_reason"); + + b.Property("RetryGroupId") + .HasColumnType("uuid") + .HasColumnName("retry_group_id"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.HasKey("Id") + .HasName("pk_xchange_result"); + + b.HasIndex("RetryGroupId") + .HasDatabaseName("ix_xchange_result_retry_group_id"); + + b.ToTable("xchange_result", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("boolean") + .HasColumnName("is_running"); + + b.ToTable("running_result", "infolink"); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("text") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("text") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.HasKey("Id") + .HasName("pk_job_executions"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("bytea") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_blob_triggers"); + + b.ToTable("qrtz_blob_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName") + .HasName("pk_qrtz_calendars"); + + b.ToTable("qrtz_calendars", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_cron_triggers"); + + b.ToTable("qrtz_cron_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("text") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId") + .HasName("pk_qrtz_fired_triggers"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_qrtz_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_qrtz_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_qrtz_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_qrtz_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_nm_gp"); + + b.ToTable("qrtz_fired_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bool") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bool") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup") + .HasName("pk_qrtz_job_details"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("qrtz_job_details", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("text") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName") + .HasName("pk_qrtz_locks"); + + b.ToTable("qrtz_locks", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup") + .HasName("pk_qrtz_paused_trigger_grps"); + + b.ToTable("qrtz_paused_trigger_grps", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName") + .HasName("pk_qrtz_scheduler_state"); + + b.ToTable("qrtz_scheduler_state", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bool") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bool") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("integer") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("integer") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("text") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("text") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("text") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_simprop_triggers"); + + b.ToTable("qrtz_simprop_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_simple_triggers"); + + b.ToTable("qrtz_simple_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("integer") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_triggers"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup") + .HasDatabaseName("ix_qrtz_triggers_sched_name_job_name_job_group"); + + b.ToTable("qrtz_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_roles_accounts_account_id"); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_roles_roles_role_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_refresh_tokens_accounts_account_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inbound_message_data_source_data_source_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_document_trail_document_document_id"); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_api_gateway_api_gateway_id"); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_subscription_subscription_id"); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .HasConstraintName("fk_bus_gateway_data_source_data_source_id"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_document_document_id"); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_route_bus_gateway_bus_gateway_id"); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_bus_gateway_route_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_route_subscription_subscription_id"); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("key"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("name"); + + b1.HasKey("PartnerId", "Id") + .HasName("pk_partner_api_credential"); + + b1.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ix_partner_api_credential_key"); + + b1.ToTable("partner_api_credential", "infolink"); + + b1.WithOwner() + .HasForeignKey("PartnerId") + .HasConstraintName("fk_partner_api_credential_partner_partner_id"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_aggregation_for"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId") + .HasConstraintName("fk_subscription_subscription_category_category_id"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_subscription_document_document_id"); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_response_subscriber"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_subscription_retry_policy_retry_policy_id"); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId") + .HasConstraintName("fk_subscription_work_group_work_group_id"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("boolean") + .HasColumnName("backwards"); + + b1.Property("On") + .HasColumnType("bigint") + .HasColumnName("on"); + + b1.Property("Recurrence") + .HasColumnType("smallint") + .HasColumnName("recurrence"); + + b1.HasKey("SubscriptionId", "Id") + .HasName("pk_subscription_schedule"); + + b1.ToTable("subscription_schedule", "infolink"); + + b1.WithOwner() + .HasForeignKey("SubscriptionId") + .HasConstraintName("fk_subscription_schedule_subscription_subscription_id"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_subscription_trail_subscription_subscription_id"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_xchange_document_document_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_aggregation_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_delivery_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_promoted_properties_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_result_xchange_id"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_blob_triggers_qrtz_triggers_sched_name_trigger_name_tr"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_cron_triggers_qrtz_triggers_sched_name_trigger_name_tr"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_simprop_triggers_qrtz_triggers_sched_name_trigger_name"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_simple_triggers_qrtz_triggers_sched_name_trigger_name_"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_triggers_qrtz_job_details_sched_name_job_name_job_group"); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.cs b/SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.cs new file mode 100644 index 00000000..42e350cb --- /dev/null +++ b/SW.Bitween.PgSql/Migrations/20260906132326_AdapterMemoryLimits.cs @@ -0,0 +1,44 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.PgSql.Migrations +{ + /// + public partial class AdapterMemoryLimits : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "hard_memory_limit_mb", + schema: "infolink", + table: "data_source", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "soft_memory_limit_mb", + schema: "infolink", + table: "data_source", + type: "integer", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "hard_memory_limit_mb", + schema: "infolink", + table: "data_source"); + + migrationBuilder.DropColumn( + name: "soft_memory_limit_mb", + schema: "infolink", + table: "data_source"); + } + } +} diff --git a/SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.Designer.cs b/SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.Designer.cs new file mode 100644 index 00000000..5714290c --- /dev/null +++ b/SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.Designer.cs @@ -0,0 +1,2641 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SW.Bitween.Model; +using SW.Bitween.PgSql; + +#nullable disable + +namespace SW.Bitween.PgSql.Migrations +{ + [DbContext(typeof(BitweenDbContext))] + [Migration("20260906141712_AdapterCpuLimits")] + partial class AdapterCpuLimits + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("infolink") + .HasAnnotation("ProductVersion", "9.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "hstore"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Deleted") + .HasColumnType("boolean") + .HasColumnName("deleted"); + + b.Property("Disabled") + .HasColumnType("boolean") + .HasColumnName("disabled"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("display_name"); + + b.Property("Email") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("email"); + + b.Property("EmailProvider") + .HasColumnType("smallint") + .HasColumnName("email_provider"); + + b.Property("FailedLoginCount") + .HasColumnType("integer") + .HasColumnName("failed_login_count"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone") + .HasColumnName("lockout_end"); + + b.Property("LoginMethods") + .HasColumnType("smallint") + .HasColumnName("login_methods"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Password") + .HasMaxLength(500) + .IsUnicode(false) + .HasColumnType("character varying(500)") + .HasColumnName("password"); + + b.Property("Role") + .HasColumnType("integer") + .HasColumnName("role"); + + b.HasKey("Id") + .HasName("pk_accounts"); + + b.HasIndex("Email") + .IsUnique() + .HasDatabaseName("ix_accounts_email"); + + b.ToTable("Accounts", "infolink"); + + b.HasData( + new + { + Id = 9999, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Deleted = false, + Disabled = false, + DisplayName = "Admin", + Email = "admin@Bitween.systems", + EmailProvider = (byte)0, + FailedLoginCount = 0, + LoginMethods = (byte)2, + Password = "$SWHASH$V1$10000$VQCi48eitH4Ml5juvBMOFZrMdQwBbhuIQVXe6RR7qJdDF2bJ", + Role = 0 + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.Property("AccountId") + .HasColumnType("integer") + .HasColumnName("account_id"); + + b.Property("RoleId") + .HasColumnType("integer") + .HasColumnName("role_id"); + + b.HasKey("AccountId", "RoleId") + .HasName("pk_account_roles"); + + b.HasIndex("RoleId") + .HasDatabaseName("ix_account_roles_role_id"); + + b.ToTable("AccountRoles", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.Property("Id") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("integer") + .HasColumnName("account_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("LoginMethod") + .HasColumnType("smallint") + .HasColumnName("login_method"); + + b.HasKey("Id") + .HasName("pk_refresh_tokens"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_refresh_tokens_account_id"); + + b.ToTable("RefreshTokens", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("description"); + + b.Property("IsSystem") + .HasColumnType("boolean") + .HasColumnName("is_system"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("Permissions") + .HasColumnType("text") + .HasColumnName("permissions"); + + b.HasKey("Id") + .HasName("pk_roles"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_roles_name"); + + b.ToTable("Roles", "infolink"); + + b.HasData( + new + { + Id = 1, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Full access to everything, including members, roles and settings.", + IsSystem = true, + Name = "Administrator", + Permissions = "[]" + }, + new + { + Id = 2, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Runs and configures integrations. Can't manage members, roles or settings.", + IsSystem = true, + Name = "Member", + Permissions = "[]" + }, + new + { + Id = 3, + CreatedOn = new DateTime(2022, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), + Description = "Read-only access to integrations, exchanges and configuration.", + IsSystem = true, + Name = "Viewer", + Permissions = "[]" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Cluster.ClusterLease", b => + { + b.Property("Id") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("id"); + + b.Property("AcquiredOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("acquired_on"); + + b.Property("OwnerNode") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("owner_node"); + + b.Property("Term") + .HasColumnType("bigint") + .HasColumnName("term"); + + b.HasKey("Id") + .HasName("pk_cluster_lease"); + + b.ToTable("cluster_lease", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdapterId") + .HasColumnType("text") + .HasColumnName("adapter_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CpuLimitSamples") + .HasColumnType("integer") + .HasColumnName("cpu_limit_samples"); + + b.Property("CpuPercentLimit") + .HasColumnType("double precision") + .HasColumnName("cpu_percent_limit"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DeduplicationWindowDays") + .HasColumnType("integer") + .HasColumnName("deduplication_window_days"); + + b.Property("HardMemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("hard_memory_limit_mb"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Kind") + .HasColumnType("integer") + .HasColumnName("kind"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("LastHeartbeatOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_heartbeat_on"); + + b.Property("LastKnownState") + .HasColumnType("text") + .HasColumnName("last_known_state"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("OwnedByNode") + .HasColumnType("text") + .HasColumnName("owned_by_node"); + + b.Property>("Properties") + .HasColumnType("hstore") + .HasColumnName("properties"); + + b.PrimitiveCollection>("SecretProperties") + .HasColumnType("text[]") + .HasColumnName("secret_properties"); + + b.Property("SoftMemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("soft_memory_limit_mb"); + + b.HasKey("Id") + .HasName("pk_data_source"); + + b.ToTable("data_source", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.Property("Id") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("id"); + + b.Property("DataSourceId") + .HasColumnType("integer") + .HasColumnName("data_source_id"); + + b.Property("SeenOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("seen_on"); + + b.Property("XchangeId") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("xchange_id"); + + b.HasKey("Id") + .HasName("pk_inbound_message"); + + b.HasIndex("DataSourceId") + .HasDatabaseName("ix_inbound_message_data_source_id"); + + b.HasIndex("SeenOn") + .HasDatabaseName("ix_inbound_message_seen_on"); + + b.ToTable("inbound_message", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DelayedRetry", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("On") + .HasColumnType("timestamp with time zone") + .HasColumnName("on"); + + b.HasKey("Id") + .HasName("pk_delayed_retry"); + + b.HasIndex("On") + .HasDatabaseName("ix_delayed_retry_on"); + + b.ToTable("delayed_retry", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusEnabled") + .HasColumnType("boolean") + .HasColumnName("bus_enabled"); + + b.Property("BusMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("bus_message_type_name"); + + b.Property("Code") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("code"); + + b.Property("DisregardsUnfilteredMessages") + .HasColumnType("boolean") + .HasColumnName("disregards_unfiltered_messages"); + + b.Property("DocumentFormat") + .HasColumnType("integer") + .HasColumnName("document_format"); + + b.Property("DuplicateInterval") + .HasColumnType("integer") + .HasColumnName("duplicate_interval"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("PromotedProperties") + .HasColumnType("jsonb") + .HasColumnName("promoted_properties"); + + b.HasKey("Id") + .HasName("pk_document"); + + b.HasIndex("BusMessageTypeName") + .IsUnique() + .HasDatabaseName("ix_document_bus_message_type_name"); + + b.HasIndex("Code") + .IsUnique() + .HasDatabaseName("ix_document_code"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_document_name"); + + b.ToTable("document", "infolink"); + + b.HasData( + new + { + Id = 10001, + BusEnabled = false, + DocumentFormat = 0, + DuplicateInterval = 0, + Name = "Aggregation Document", + PromotedProperties = "{}" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("Code") + .HasColumnType("integer") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("StateAfter") + .HasColumnType("text") + .HasColumnName("state_after"); + + b.Property("StateBefore") + .HasColumnType("text") + .HasColumnName("state_before"); + + b.HasKey("Id") + .HasName("pk_document_trail"); + + b.HasIndex("CreatedOn") + .HasDatabaseName("ix_document_trail_created_on"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_document_trail_document_id"); + + b.ToTable("document_trail", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("UrlName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("url_name"); + + b.HasKey("Id") + .HasName("pk_api_gateway"); + + b.HasIndex("UrlName") + .IsUnique() + .HasDatabaseName("ix_api_gateway_url_name"); + + b.ToTable("api_gateway", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.Property("ApiGatewayId") + .HasColumnType("integer") + .HasColumnName("api_gateway_id"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.HasKey("ApiGatewayId", "PartnerId", "SubscriptionId") + .HasName("pk_api_gateway_partner"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_api_gateway_partner_partner_id"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_api_gateway_partner_subscription_id"); + + b.ToTable("api_gateway_partner", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DataSourceId") + .HasColumnType("integer") + .HasColumnName("data_source_id"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property>("EndpointProperties") + .HasColumnType("hstore") + .HasColumnName("endpoint_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_bus_gateway"); + + b.HasIndex("DataSourceId") + .HasDatabaseName("ix_bus_gateway_data_source_id"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_bus_gateway_document_id"); + + b.ToTable("bus_gateway", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusGatewayId") + .HasColumnType("integer") + .HasColumnName("bus_gateway_id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("MatchExpression") + .HasColumnType("text") + .HasColumnName("match_expression"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_bus_gateway_route"); + + b.HasIndex("BusGatewayId") + .HasDatabaseName("ix_bus_gateway_route_bus_gateway_id"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_bus_gateway_route_partner_id"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_bus_gateway_route_subscription_id"); + + b.ToTable("bus_gateway_route", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.GlobalAdapterValuesSet", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property>("Values") + .HasColumnType("jsonb") + .HasColumnName("values"); + + b.HasKey("Id") + .HasName("pk_global_adapter_values_set"); + + b.ToTable("global_adapter_values_set", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Notifier", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("HandlerId") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property("HandlerProperties") + .HasColumnType("text") + .HasColumnName("handler_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("RunOnBadResult") + .HasColumnType("boolean") + .HasColumnName("run_on_bad_result"); + + b.Property("RunOnFailedResult") + .HasColumnType("boolean") + .HasColumnName("run_on_failed_result"); + + b.PrimitiveCollection("RunOnSubscriptions") + .HasColumnType("integer[]") + .HasColumnName("run_on_subscriptions"); + + b.Property("RunOnSuccessfulResult") + .HasColumnType("boolean") + .HasColumnName("run_on_successful_result"); + + b.HasKey("Id") + .HasName("pk_notifier"); + + b.ToTable("notifier", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.OnHoldXchange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BadData") + .HasColumnType("boolean") + .HasColumnName("bad_data"); + + b.Property("Data") + .HasColumnType("text") + .HasColumnName("data"); + + b.Property("FileName") + .HasColumnType("text") + .HasColumnName("file_name"); + + b.PrimitiveCollection("References") + .HasColumnType("text[]") + .HasColumnName("references"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_on_hold_xchange"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_on_hold_xchange_subscription_id"); + + b.ToTable("on_hold_xchange", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property>("AdapterProperties") + .HasColumnType("jsonb") + .HasColumnName("adapter_properties"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_partner"); + + b.ToTable("partner", "infolink"); + + b.HasData( + new + { + Id = 1, + Name = "SYSTEM" + }); + }); + + modelBuilder.Entity("SW.Bitween.Domain.ReceiveAttempt", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasColumnType("text") + .HasColumnName("error_message"); + + b.PrimitiveCollection("ExchangeIds") + .HasColumnType("text[]") + .HasColumnName("exchange_ids"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("Outcome") + .HasColumnType("integer") + .HasColumnName("outcome"); + + b.Property("StartedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_on"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_receive_attempt"); + + b.HasIndex("SubscriptionId", "StartedOn") + .HasDatabaseName("ix_receive_attempt_subscription_id_started_on"); + + b.ToTable("receive_attempt", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryAlertOverride", b => + { + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("GroupId") + .HasColumnType("uuid") + .HasColumnName("group_id"); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("alert_handler_id"); + + b.Property("AlertHandlerProperties") + .HasColumnType("text") + .HasColumnName("alert_handler_properties"); + + b.Property("AlertMode") + .HasColumnType("smallint") + .HasColumnName("alert_mode"); + + b.HasKey("SubscriptionId", "GroupId") + .HasName("pk_retry_alert_override"); + + b.ToTable("retry_alert_override", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryGroupUsage", b => + { + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.Property("GroupId") + .HasColumnType("uuid") + .HasColumnName("group_id"); + + b.Property("AttemptsUsed") + .HasColumnType("integer") + .HasColumnName("attempts_used"); + + b.Property("ExhaustedNotifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("exhausted_notified_on"); + + b.Property("LastAttemptOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempt_on"); + + b.HasKey("SubscriptionId", "GroupId") + .HasName("pk_retry_group_usage"); + + b.ToTable("retry_group_usage", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.RetryPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AlertHandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("alert_handler_id"); + + b.Property("AlertHandlerProperties") + .HasColumnType("text") + .HasColumnName("alert_handler_properties"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Groups") + .HasColumnType("text") + .HasColumnName("groups"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_retry_policy"); + + b.ToTable("retry_policy", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Setting", b => + { + b.Property("Id") + .HasMaxLength(200) + .IsUnicode(false) + .HasColumnType("character varying(200)") + .HasColumnName("id"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.Property("Value") + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Id") + .HasName("pk_settings"); + + b.ToTable("Settings", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AggregateOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("aggregate_on"); + + b.Property("AggregationForId") + .HasColumnType("integer") + .HasColumnName("aggregation_for_id"); + + b.Property("AggregationTarget") + .HasColumnType("smallint") + .HasColumnName("aggregation_target"); + + b.Property("CategoryId") + .HasColumnType("integer") + .HasColumnName("category_id"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer") + .HasColumnName("consecutive_failures"); + + b.Property("CustomRetryPolicy") + .HasColumnType("text") + .HasColumnName("custom_retry_policy"); + + b.Property>("DocumentFilter") + .HasColumnType("jsonb") + .HasColumnName("document_filter"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("HandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property>("HandlerProperties") + .HasColumnType("jsonb") + .HasColumnName("handler_properties"); + + b.Property("Inactive") + .HasColumnType("boolean") + .HasColumnName("inactive"); + + b.Property("IsRunning") + .HasColumnType("boolean") + .HasColumnName("is_running"); + + b.Property("LastException") + .HasColumnType("text") + .HasColumnName("last_exception"); + + b.Property("MapperId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("mapper_id"); + + b.Property>("MapperProperties") + .HasColumnType("jsonb") + .HasColumnName("mapper_properties"); + + b.Property("MatchExpression") + .HasColumnType("text") + .HasColumnName("match_expression"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.Property("PausedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("paused_on"); + + b.Property("ReceiveOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("receive_on"); + + b.Property("ReceiverId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("receiver_id"); + + b.Property>("ReceiverProperties") + .HasColumnType("jsonb") + .HasColumnName("receiver_properties"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("response_message_type_name"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("integer") + .HasColumnName("response_subscription_id"); + + b.Property("RetryPolicyId") + .HasColumnType("integer") + .HasColumnName("retry_policy_id"); + + b.Property("Temporary") + .HasColumnType("boolean") + .HasColumnName("temporary"); + + b.Property("Type") + .HasColumnType("smallint") + .HasColumnName("type"); + + b.Property("ValidatorId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("validator_id"); + + b.Property>("ValidatorProperties") + .HasColumnType("jsonb") + .HasColumnName("validator_properties"); + + b.Property("WorkGroupId") + .HasColumnType("integer") + .HasColumnName("work_group_id"); + + b.HasKey("Id") + .HasName("pk_subscription"); + + b.HasIndex("AggregationForId") + .HasDatabaseName("ix_subscription_aggregation_for_id"); + + b.HasIndex("CategoryId") + .HasDatabaseName("ix_subscription_category_id"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_subscription_document_id"); + + b.HasIndex("PartnerId") + .HasDatabaseName("ix_subscription_partner_id"); + + b.HasIndex("ResponseSubscriptionId") + .HasDatabaseName("ix_subscription_response_subscription_id"); + + b.HasIndex("RetryPolicyId") + .HasDatabaseName("ix_subscription_retry_policy_id"); + + b.HasIndex("WorkGroupId") + .HasDatabaseName("ix_subscription_work_group_id"); + + b.ToTable("subscription", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionCategory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("text") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("ModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified_on"); + + b.HasKey("Id") + .HasName("pk_subscription_category"); + + b.HasIndex("Code") + .IsUnique() + .HasDatabaseName("ix_subscription_category_code"); + + b.ToTable("subscription_category", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("Code") + .HasColumnType("integer") + .HasColumnName("code"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("StateAfter") + .HasColumnType("text") + .HasColumnName("state_after"); + + b.Property("StateBefore") + .HasColumnType("text") + .HasColumnName("state_before"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_subscription_trail"); + + b.HasIndex("CreatedOn") + .HasDatabaseName("ix_subscription_trail_created_on"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_subscription_trail_subscription_id"); + + b.ToTable("subscription_trail", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.WorkGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BusMessageName") + .IsRequired() + .HasMaxLength(100) + .IsUnicode(false) + .HasColumnType("character varying(100)") + .HasColumnName("bus_message_name"); + + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Options") + .HasColumnType("jsonb") + .HasColumnName("options"); + + b.HasKey("Id") + .HasName("pk_work_group"); + + b.ToTable("work_group", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("CorrelationId") + .HasColumnType("text") + .HasColumnName("correlation_id"); + + b.Property("DocumentId") + .HasColumnType("integer") + .HasColumnName("document_id"); + + b.Property("HandlerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("handler_id"); + + b.Property>("HandlerProperties") + .HasColumnType("jsonb") + .HasColumnName("handler_properties"); + + b.Property("InputContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("input_content_type"); + + b.Property("InputHash") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("input_hash"); + + b.Property("InputName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("input_name"); + + b.Property("InputSize") + .HasColumnType("integer") + .HasColumnName("input_size"); + + b.Property("ManualRetry") + .HasColumnType("boolean") + .HasColumnName("manual_retry"); + + b.Property("MapperId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("mapper_id"); + + b.Property>("MapperProperties") + .HasColumnType("jsonb") + .HasColumnName("mapper_properties"); + + b.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b.PrimitiveCollection("References") + .HasColumnType("text[]") + .HasColumnName("references"); + + b.Property("ResponseMessageTypeName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("response_message_type_name"); + + b.Property("ResponseSubscriptionId") + .HasColumnType("integer") + .HasColumnName("response_subscription_id"); + + b.Property("RetryFor") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("retry_for"); + + b.Property("StartedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_on"); + + b.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b.HasKey("Id") + .HasName("pk_xchange"); + + b.HasIndex("DocumentId") + .HasDatabaseName("ix_xchange_document_id"); + + b.HasIndex("InputHash") + .HasDatabaseName("ix_xchange_input_hash"); + + b.HasIndex("RetryFor") + .HasDatabaseName("ix_xchange_retry_for"); + + b.HasIndex("StartedOn") + .HasDatabaseName("ix_xchange_started_on"); + + b.HasIndex("SubscriptionId") + .HasDatabaseName("ix_xchange_subscription_id"); + + b.ToTable("xchange", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AggregatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("aggregated_on"); + + b.Property("AggregationXchangeId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("aggregation_xchange_id"); + + b.HasKey("Id") + .HasName("pk_xchange_aggregation"); + + b.HasIndex("AggregationXchangeId") + .HasDatabaseName("ix_xchange_aggregation_aggregation_xchange_id"); + + b.ToTable("xchange_aggregation", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("DeliveredOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("delivered_on"); + + b.HasKey("Id") + .HasName("pk_xchange_delivery"); + + b.HasIndex("DeliveredOn") + .HasDatabaseName("ix_xchange_delivery_delivered_on"); + + b.ToTable("xchange_delivery", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("text") + .HasColumnName("exception"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("NotifierId") + .HasColumnType("integer") + .HasColumnName("notifier_id"); + + b.Property("NotifierName") + .HasColumnType("text") + .HasColumnName("notifier_name"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.Property("XchangeId") + .HasMaxLength(50) + .IsUnicode(false) + .HasColumnType("character varying(50)") + .HasColumnName("xchange_id"); + + b.HasKey("Id") + .HasName("pk_xchange_notification"); + + b.ToTable("xchange_notification", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.PrimitiveCollection("Hits") + .HasColumnType("integer[]") + .HasColumnName("hits"); + + b.Property>("Properties") + .HasColumnType("jsonb") + .HasColumnName("properties"); + + b.Property("PropertiesRaw") + .HasColumnType("text") + .HasColumnName("properties_raw"); + + b.HasKey("Id") + .HasName("pk_xchange_promoted_properties"); + + b.HasIndex("PropertiesRaw") + .HasDatabaseName("ix_xchange_promoted_properties_properties_raw"); + + b.ToTable("xchange_promoted_properties", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("id"); + + b.Property("AttemptNumber") + .HasColumnType("integer") + .HasColumnName("attempt_number"); + + b.Property("Exception") + .HasColumnType("text") + .HasColumnName("exception"); + + b.Property("FinishedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_on"); + + b.Property("OutputBad") + .HasColumnType("boolean") + .HasColumnName("output_bad"); + + b.Property("OutputContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("output_content_type"); + + b.Property("OutputHash") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("output_hash"); + + b.Property("OutputName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("output_name"); + + b.Property("OutputSize") + .HasColumnType("integer") + .HasColumnName("output_size"); + + b.Property("ResponseBad") + .HasColumnType("boolean") + .HasColumnName("response_bad"); + + b.Property("ResponseContentType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("response_content_type"); + + b.Property("ResponseHash") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("response_hash"); + + b.Property("ResponseName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("response_name"); + + b.Property("ResponseSize") + .HasColumnType("integer") + .HasColumnName("response_size"); + + b.Property("ResponseXchangeId") + .HasColumnType("text") + .HasColumnName("response_xchange_id"); + + b.Property("RetryBlockedReason") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("retry_blocked_reason"); + + b.Property("RetryGroupId") + .HasColumnType("uuid") + .HasColumnName("retry_group_id"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.HasKey("Id") + .HasName("pk_xchange_result"); + + b.HasIndex("RetryGroupId") + .HasDatabaseName("ix_xchange_result_retry_group_id"); + + b.ToTable("xchange_result", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.RunFlagUpdater+RunningResult", b => + { + b.Property("IsRunning") + .HasColumnType("boolean") + .HasColumnName("is_running"); + + b.ToTable("running_result", "infolink"); + + b.ToView(null, (string)null); + }); + + modelBuilder.Entity("SW.Scheduler.JobExecution", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("EndTimeUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("end_time_utc"); + + b.Property("Error") + .HasColumnType("text") + .HasColumnName("error"); + + b.Property("FireInstanceId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("fire_instance_id"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobTypeName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_type_name"); + + b.Property("Node") + .IsRequired() + .HasColumnType("text") + .HasColumnName("node"); + + b.Property("StartTimeUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time_utc"); + + b.Property("Success") + .HasColumnType("boolean") + .HasColumnName("success"); + + b.HasKey("Id") + .HasName("pk_job_executions"); + + b.HasIndex("FireInstanceId") + .IsUnique() + .HasDatabaseName("idx_je_fire_instance_id"); + + b.HasIndex("StartTimeUtc") + .HasDatabaseName("idx_je_start_time"); + + b.HasIndex("Success") + .HasDatabaseName("idx_je_success"); + + b.HasIndex("JobGroup", "JobName", "StartTimeUtc") + .HasDatabaseName("idx_je_group_name_start"); + + b.ToTable("job_executions", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("bytea") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_blob_triggers"); + + b.ToTable("qrtz_blob_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName") + .HasName("pk_qrtz_calendars"); + + b.ToTable("qrtz_calendars", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_cron_triggers"); + + b.ToTable("qrtz_cron_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("text") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId") + .HasName("pk_qrtz_fired_triggers"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_qrtz_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_qrtz_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_qrtz_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_qrtz_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_nm_gp"); + + b.ToTable("qrtz_fired_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bool") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bool") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup") + .HasName("pk_qrtz_job_details"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_j_req_recovery"); + + b.ToTable("qrtz_job_details", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("text") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName") + .HasName("pk_qrtz_locks"); + + b.ToTable("qrtz_locks", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup") + .HasName("pk_qrtz_paused_trigger_grps"); + + b.ToTable("qrtz_paused_trigger_grps", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName") + .HasName("pk_qrtz_scheduler_state"); + + b.ToTable("qrtz_scheduler_state", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bool") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bool") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("integer") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("integer") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("text") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("text") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("text") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_simprop_triggers"); + + b.ToTable("qrtz_simprop_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_simple_triggers"); + + b.ToTable("qrtz_simple_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("integer") + .HasColumnName("misfire_instr"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup") + .HasName("pk_qrtz_triggers"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup") + .HasDatabaseName("ix_qrtz_triggers_sched_name_job_name_job_group"); + + b.ToTable("qrtz_triggers", "infolink"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.AccountRoleLink", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_roles_accounts_account_id"); + + b.HasOne("SW.Bitween.Domain.Accounts.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_roles_roles_role_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Accounts.RefreshToken", b => + { + b.HasOne("SW.Bitween.Domain.Accounts.Account", null) + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_refresh_tokens_accounts_account_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DataSources.InboundMessage", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", null) + .WithMany() + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inbound_message_data_source_data_source_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.DocumentTrail", b => + { + b.HasOne("SW.Bitween.Domain.Document", "Document") + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_document_trail_document_document_id"); + + b.Navigation("Document"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGatewayPartner", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.ApiGateway", "ApiGateway") + .WithMany("Partners") + .HasForeignKey("ApiGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_api_gateway_api_gateway_id"); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_api_gateway_partner_subscription_subscription_id"); + + b.Navigation("ApiGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.HasOne("SW.Bitween.Domain.DataSources.DataSource", "DataSource") + .WithMany() + .HasForeignKey("DataSourceId") + .HasConstraintName("fk_bus_gateway_data_source_data_source_id"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_document_document_id"); + + b.Navigation("DataSource"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGatewayRoute", b => + { + b.HasOne("SW.Bitween.Domain.Gateway.BusGateway", "BusGateway") + .WithMany("Routes") + .HasForeignKey("BusGatewayId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_route_bus_gateway_bus_gateway_id"); + + b.HasOne("SW.Bitween.Domain.Partner", "Partner") + .WithMany() + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_bus_gateway_route_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_bus_gateway_route_subscription_subscription_id"); + + b.Navigation("BusGateway"); + + b.Navigation("Partner"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.OwnsMany("SW.Bitween.Domain.ApiCredential", "ApiCredentials", b1 => + { + b1.Property("PartnerId") + .HasColumnType("integer") + .HasColumnName("partner_id"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b1.Property("Id")); + + b1.Property("Key") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("key"); + + b1.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("name"); + + b1.HasKey("PartnerId", "Id") + .HasName("pk_partner_api_credential"); + + b1.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ix_partner_api_credential_key"); + + b1.ToTable("partner_api_credential", "infolink"); + + b1.WithOwner() + .HasForeignKey("PartnerId") + .HasConstraintName("fk_partner_api_credential_partner_partner_id"); + + b1.HasData( + new + { + PartnerId = 1, + Id = 1, + Key = "7facc758283844b49cc4ffd26a75b1de", + Name = "default" + }); + }); + + b.Navigation("ApiCredentials"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Subscription", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("AggregationForId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_aggregation_for"); + + b.HasOne("SW.Bitween.Domain.SubscriptionCategory", "Category") + .WithMany() + .HasForeignKey("CategoryId") + .HasConstraintName("fk_subscription_subscription_category_category_id"); + + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_subscription_document_document_id"); + + b.HasOne("SW.Bitween.Domain.Partner", null) + .WithMany("Subscriptions") + .HasForeignKey("PartnerId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_partner_partner_id"); + + b.HasOne("SW.Bitween.Domain.Subscription", null) + .WithMany() + .HasForeignKey("ResponseSubscriptionId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_subscription_response_subscriber"); + + b.HasOne("SW.Bitween.Domain.RetryPolicy", "RetryPolicy") + .WithMany() + .HasForeignKey("RetryPolicyId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_subscription_retry_policy_retry_policy_id"); + + b.HasOne("SW.Bitween.Domain.WorkGroup", "WorkGroup") + .WithMany() + .HasForeignKey("WorkGroupId") + .HasConstraintName("fk_subscription_work_group_work_group_id"); + + b.OwnsMany("SW.Bitween.Domain.Schedule", "Schedules", b1 => + { + b1.Property("SubscriptionId") + .HasColumnType("integer") + .HasColumnName("subscription_id"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b1.Property("Id")); + + b1.Property("Backwards") + .HasColumnType("boolean") + .HasColumnName("backwards"); + + b1.Property("On") + .HasColumnType("bigint") + .HasColumnName("on"); + + b1.Property("Recurrence") + .HasColumnType("smallint") + .HasColumnName("recurrence"); + + b1.HasKey("SubscriptionId", "Id") + .HasName("pk_subscription_schedule"); + + b1.ToTable("subscription_schedule", "infolink"); + + b1.WithOwner() + .HasForeignKey("SubscriptionId") + .HasConstraintName("fk_subscription_schedule_subscription_subscription_id"); + }); + + b.Navigation("Category"); + + b.Navigation("RetryPolicy"); + + b.Navigation("Schedules"); + + b.Navigation("WorkGroup"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.SubscriptionTrail", b => + { + b.HasOne("SW.Bitween.Domain.Subscription", "Subscription") + .WithMany() + .HasForeignKey("SubscriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_subscription_trail_subscription_subscription_id"); + + b.Navigation("Subscription"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Xchange", b => + { + b.HasOne("SW.Bitween.Domain.Document", null) + .WithMany() + .HasForeignKey("DocumentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_xchange_document_document_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeAggregation", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeAggregation", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_aggregation_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeDelivery", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeDelivery", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_delivery_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangePromotedProperties", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangePromotedProperties", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_promoted_properties_xchange_id"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.XchangeResult", b => + { + b.HasOne("SW.Bitween.Domain.Xchange", null) + .WithOne() + .HasForeignKey("SW.Bitween.Domain.XchangeResult", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_xchange_result_xchange_id"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzBlobTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_blob_triggers_qrtz_triggers_sched_name_trigger_name_tr"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzCronTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_cron_triggers_qrtz_triggers_sched_name_trigger_name_tr"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimplePropertyTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_simprop_triggers_qrtz_triggers_sched_name_trigger_name"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzSimpleTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_simple_triggers_qrtz_triggers_sched_name_trigger_name_"); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.HasOne("SW.Scheduler.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qrtz_triggers_qrtz_job_details_sched_name_job_name_job_group"); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.ApiGateway", b => + { + b.Navigation("Partners"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Gateway.BusGateway", b => + { + b.Navigation("Routes"); + }); + + modelBuilder.Entity("SW.Bitween.Domain.Partner", b => + { + b.Navigation("Subscriptions"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("SW.Scheduler.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.cs b/SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.cs new file mode 100644 index 00000000..5172aeec --- /dev/null +++ b/SW.Bitween.PgSql/Migrations/20260906141712_AdapterCpuLimits.cs @@ -0,0 +1,44 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SW.Bitween.PgSql.Migrations +{ + /// + public partial class AdapterCpuLimits : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "cpu_limit_samples", + schema: "infolink", + table: "data_source", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "cpu_percent_limit", + schema: "infolink", + table: "data_source", + type: "double precision", + nullable: false, + defaultValue: 0.0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "cpu_limit_samples", + schema: "infolink", + table: "data_source"); + + migrationBuilder.DropColumn( + name: "cpu_percent_limit", + schema: "infolink", + table: "data_source"); + } + } +} diff --git a/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs b/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs index 273354b3..26ed18ba 100644 --- a/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs +++ b/SW.Bitween.PgSql/Migrations/BitweenDbContextModelSnapshot.cs @@ -297,6 +297,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("consecutive_failures"); + b.Property("CpuLimitSamples") + .HasColumnType("integer") + .HasColumnName("cpu_limit_samples"); + + b.Property("CpuPercentLimit") + .HasColumnType("double precision") + .HasColumnName("cpu_percent_limit"); + b.Property("CreatedBy") .HasColumnType("text") .HasColumnName("created_by"); @@ -309,6 +317,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("deduplication_window_days"); + b.Property("HardMemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("hard_memory_limit_mb"); + b.Property("Inactive") .HasColumnType("boolean") .HasColumnName("inactive"); @@ -353,6 +365,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("text[]") .HasColumnName("secret_properties"); + b.Property("SoftMemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("soft_memory_limit_mb"); + b.HasKey("Id") .HasName("pk_data_source"); diff --git a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj index d360c7c1..d93417b5 100644 --- a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj +++ b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj @@ -5,6 +5,6 @@ SW.Bitween.SampleConfigurableAdapter - + diff --git a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj index 0e8b3ed8..f51ac2d1 100644 --- a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj +++ b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj index 4bff15fa..165bed0d 100644 --- a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj +++ b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj index 7af762f1..76436d7b 100644 --- a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj +++ b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj @@ -8,7 +8,7 @@ - + diff --git a/SW.Bitween.Sdk/Model/DataSource.cs b/SW.Bitween.Sdk/Model/DataSource.cs index d31b4536..8edf2b8c 100644 --- a/SW.Bitween.Sdk/Model/DataSource.cs +++ b/SW.Bitween.Sdk/Model/DataSource.cs @@ -30,6 +30,28 @@ public class DataSourceCreate : IName public bool Inactive { get; set; } + /// + /// Soft ceiling in MB for the adapter process. Crossing it recycles the adapter between + /// messages rather than killing it. Zero leaves the host default. + /// + public int SoftMemoryLimitMb { get; set; } + + /// + /// Hard ceiling in MB, enforced by the runtime as the adapter's GC heap hard limit, so an + /// allocation past it fails inside the adapter rather than taking the node with it. Zero + /// leaves the host default. + /// + public int HardMemoryLimitMb { get; set; } + + /// + /// Sustained CPU ceiling as a percentage of the whole node. Zero leaves the host default. + /// One pegged core on a sixteen-core node is about 6%, not 100% — see the entity's remarks. + /// + public double CpuPercentLimit { get; set; } + + /// Consecutive heartbeats above the CPU ceiling before it trips. Zero = host default. + public int CpuLimitSamples { get; set; } + /// /// How long a message's dedupe key is remembered. It has to exceed the widest redelivery window /// this broker can produce. Zero turns deduplication off. @@ -144,3 +166,25 @@ public class DataSourceTelemetry /// What the adapter says it can do — the commands the UI could offer against it. public List Commands { get; set; } = new(); } + +/// Which read-only command to relay to the running adapter. Defaults to Discover. +public class DataSourceInspectRequest +{ + public string Command { get; set; } +} + +public class DataSourceInspectResult +{ + /// False when this node is not the one holding the connection, or the command failed. + public bool Ran { get; set; } + + public string Command { get; set; } + + /// + /// The adapter's answer, as JSON text. Untyped for the same reason its telemetry is: Bitween + /// does not model any broker's topology, and a provider must be free to describe its own. + /// + public string Result { get; set; } + + public string Error { get; set; } +} diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index a6c5a031..23a17566 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -11,6 +11,7 @@ import type { BusGatewayDetail, BusGatewayRow, DataSourceDetail, + DataSourceInspectResult, DataSourceRow, DataSourceTelemetry, DataSourceTestResult, @@ -335,11 +336,17 @@ export interface ApiClient { secretProperties: string[]; inactive: boolean; deduplicationWindowDays: number; + softMemoryLimitMb: number; + hardMemoryLimitMb: number; + cpuPercentLimit: number; + cpuLimitSamples: number; }, ): Promise; deleteDataSource(id: number): Promise; testDataSource(id: number): Promise; getDataSourceTelemetry(id: number): Promise; + /** Relays a read-only command (Discover, GetStats) to the adapter actually serving traffic. */ + inspectDataSource(id: number, command: string): Promise; /** The subscription is either an existing id or defined inline; the endpoint commits both as one. */ addBusRoute(id: number, input: AddBusRouteInput): Promise; updateBusRoute( diff --git a/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts b/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts index 36065e40..20a5f81b 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts @@ -1,6 +1,7 @@ import type { ApiClient } from "../client"; import type { DataSourceDetail, + DataSourceInspectResult, DataSourceRow, DataSourceTelemetry, DataSourceTestResult, @@ -20,6 +21,10 @@ interface RawDataSource { kind: string; inactive: boolean | null; deduplicationWindowDays: number; + softMemoryLimitMb?: number | null; + hardMemoryLimitMb?: number | null; + cpuPercentLimit?: number | null; + cpuLimitSamples?: number | null; gatewayCount: number; lastKnownState: string | null; lastHeartbeatOn: string | null; @@ -50,6 +55,10 @@ const toRow = (raw: RawDataSource): DataSourceRow => ({ kind: raw.kind, inactive: raw.inactive ?? false, deduplicationWindowDays: raw.deduplicationWindowDays, + softMemoryLimitMb: raw.softMemoryLimitMb ?? 0, + hardMemoryLimitMb: raw.hardMemoryLimitMb ?? 0, + cpuPercentLimit: raw.cpuPercentLimit ?? 0, + cpuLimitSamples: raw.cpuLimitSamples ?? 0, gatewayCount: raw.gatewayCount, lastKnownState: raw.lastKnownState, lastHeartbeatOn: raw.lastHeartbeatOn, @@ -109,6 +118,10 @@ export const dataSourceMethods = { secretProperties: input.secretProperties, inactive: false, deduplicationWindowDays: 30, + softMemoryLimitMb: 0, + hardMemoryLimitMb: 0, + cpuPercentLimit: 0, + cpuLimitSamples: 0, }); return { id }; }, @@ -123,6 +136,10 @@ export const dataSourceMethods = { secretProperties: string[]; inactive: boolean; deduplicationWindowDays: number; + softMemoryLimitMb: number; + hardMemoryLimitMb: number; + cpuPercentLimit: number; + cpuLimitSamples: number; }, ): Promise { await post(`/datasources/${id}`, changes); @@ -132,6 +149,10 @@ export const dataSourceMethods = { await request(`/datasources/${id}`, { method: "DELETE" }); }, + async inspectDataSource(id: number, command: string): Promise { + return post(`/datasources/${id}/inspect`, { command }); + }, + /** Live, from the heartbeat. Scoped to the node that answers — see the backend handler. */ async getDataSourceTelemetry(id: number): Promise { const raw = await get(`/datasources/${id}/telemetry`); diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index 26ec17ca..16cdcc64 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -763,6 +763,19 @@ export interface DataSource { */ deduplicationWindowDays: number; + /** Soft ceiling in MB — crossing it recycles the adapter between messages. 0 = host default. */ + softMemoryLimitMb: number; + /** Hard ceiling in MB — becomes the adapter's GC heap hard limit. 0 = host default. */ + hardMemoryLimitMb: number; + + /** + * Sustained CPU ceiling as a share of the WHOLE node, not of one core. One pegged core on a + * sixteen-core node is about 6%. 0 = host default. + */ + cpuPercentLimit: number; + /** Consecutive heartbeats above the ceiling before it trips. 0 = host default. */ + cpuLimitSamples: number; + // Health, written back by the supervisor from the adapter's heartbeat. lastKnownState: string | null; lastHeartbeatOn: string | null; @@ -816,6 +829,15 @@ export interface DataSourceTelemetry { commands: string[]; } +/** What a read-only command relayed to the running adapter came back with. */ +export interface DataSourceInspectResult { + ran: boolean; + command: string | null; + /** The adapter's own JSON, as text — Bitween does not model any broker's topology. */ + result: string | null; + error: string | null; +} + export interface DataSourceTestStage { name: string; succeeded: boolean; diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx index c74635ef..a09af9d4 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourcePage.tsx @@ -1,8 +1,15 @@ import { useEffect, useState } from "react"; import { Link, useNavigate, useParams } from "react-router"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Check, Plug, Plus, Trash2, X } from "lucide-react"; -import { api, ApiRequestError, SECRET_SENTINEL, type DataSourceDetail, type DataSourceTestResult } from "../../api"; +import { Check, Gauge, Plug, Plus, Telescope, Trash2, X } from "lucide-react"; +import { + api, + ApiRequestError, + SECRET_SENTINEL, + type DataSourceDetail, + type DataSourceInspectResult, + type DataSourceTestResult, +} from "../../api"; import { Can, useSessionCan } from "../../auth/guards"; import { PageHeader } from "../../components/layout/PageHeader"; import { Badge, Button, FormError, LoadingBlock } from "../../components/ui/basics"; @@ -19,6 +26,10 @@ interface Draft { name: string; inactive: boolean; deduplicationWindowDays: number; + softMemoryLimitMb: number; + hardMemoryLimitMb: number; + cpuPercentLimit: number; + cpuLimitSamples: number; properties: Record; } @@ -26,6 +37,10 @@ const draftOf = (d: DataSourceDetail): Draft => ({ name: d.name, inactive: d.inactive, deduplicationWindowDays: d.deduplicationWindowDays, + softMemoryLimitMb: d.softMemoryLimitMb, + hardMemoryLimitMb: d.hardMemoryLimitMb, + cpuPercentLimit: d.cpuPercentLimit, + cpuLimitSamples: d.cpuLimitSamples, properties: { ...d.properties }, }); @@ -54,6 +69,7 @@ export function DataSourcePage() { const [error, setError] = useState(null); const [result, setResult] = useState(null); const [newKey, setNewKey] = useState(""); + const [inspect, setInspect] = useState(null); const [removing, setRemoving] = useState(false); // Re-seed whenever the server's copy changes: saving re-masks the secrets, so the form has to @@ -72,6 +88,10 @@ export function DataSourcePage() { secretProperties: source.data!.secretProperties, inactive: d.inactive, deduplicationWindowDays: d.deduplicationWindowDays, + softMemoryLimitMb: d.softMemoryLimitMb, + hardMemoryLimitMb: d.hardMemoryLimitMb, + cpuPercentLimit: d.cpuPercentLimit, + cpuLimitSamples: d.cpuLimitSamples, }), onSuccess: async () => { setError(null); @@ -91,6 +111,20 @@ export function DataSourcePage() { }), }); + // Discover and GetStats go to the adapter actually serving traffic, not to a throwaway instance + // the way Test does — they are questions about the live connection. + const ask = useMutation({ + mutationFn: (command: string) => api.inspectDataSource(dataSourceId, command), + onSuccess: setInspect, + onError: (e) => + setInspect({ + ran: false, + command: null, + result: null, + error: e instanceof ApiRequestError ? e.message : "The command could not be run.", + }), + }); + const remove = useMutation({ mutationFn: () => api.deleteDataSource(dataSourceId), onSuccess: async () => { @@ -108,6 +142,10 @@ export function DataSourcePage() { return This data source no longer exists.; const d = source.data; + + // Only ever used to illustrate what a percentage means. This is the BROWSER's core count, not + // the node's, so it is a rough translation rather than a claim about the server. + const cores = navigator.hardwareConcurrency || 8; const provider = providerOf(d.adapterId); const dirty = JSON.stringify(draft) !== JSON.stringify(draftOf(d)); @@ -155,6 +193,12 @@ export function DataSourcePage() { {test.isPending ? "Testing…" : "Test connection"} + + + + + {inspect.ran ? ( + 
+              {inspect.result}
+
+ ) : ( +

{inspect.error}

+ )} + +

+ Asked of the connection that is actually serving traffic, not a throwaway one — and + read-only: nothing is consumed, acknowledged or published. +

+ + )} + {/* ——— the test's answer ——— */} {result && (
+
+

Memory ceilings

+

+ An adapter is a separate process holding this broker's connection. Without a ceiling + it is bounded by nothing but the host, so one runaway payload takes every other + integration on the node down with it. Leave both at 0 to use the host's own defaults. +

+ +
+
+ + + setDraft({ ...draft, softMemoryLimitMb: Number(e.target.value) || 0 }) + } + /> + +
+
+ + + setDraft({ ...draft, hardMemoryLimitMb: Number(e.target.value) || 0 }) + } + /> + +
+
+ + {draft.softMemoryLimitMb > 0 && + draft.hardMemoryLimitMb > 0 && + draft.softMemoryLimitMb > draft.hardMemoryLimitMb && ( +

+ A soft limit above the hard one can never be reached — the runtime fails the + allocation first, so the recycle never happens. +

+ )} + +

+ Applied when the adapter process launches, so changing these restarts it. Nothing in + flight is lost: messages are only acknowledged once Bitween has persisted them. +

+
+ +
+

CPU ceiling

+

+ A share of the whole node, not of one core — one core pegged flat out + on a sixteen-core node reads about 6%, so “50%” would allow eight cores + rather than half of one. It trips only after several consecutive heartbeats above the + line, because an adapter draining a backlog is supposed to work hard. + Leave at 0 for the host default. +

+ +
+
+ 0 + ? `For scale: ${(draft.cpuPercentLimit / 100 * cores).toFixed(1)} core(s) on a ` + + `${cores}-core machine — this browser's core count, not the node's.` + : "Off — the host default applies." + } + > + + setDraft({ ...draft, cpuPercentLimit: Number(e.target.value) || 0 }) + } + /> + +
+
+ + + setDraft({ ...draft, cpuLimitSamples: Number(e.target.value) || 0 }) + } + /> + +
+
+ + {draft.cpuPercentLimit > 100 && ( +

+ Above 100% can never be reached — the figure is a share of the whole node, so 100% + is every core at once. +

+ )} + +

+ Crossing it asks the adapter to drain rather than killing it, so in-flight messages go + back to the broker instead of being lost. +

+
+

Connection settings

diff --git a/SW.Bitween.Web/SW.Bitween.Web.csproj b/SW.Bitween.Web/SW.Bitween.Web.csproj index 220e9944..18562abd 100644 --- a/SW.Bitween.Web/SW.Bitween.Web.csproj +++ b/SW.Bitween.Web/SW.Bitween.Web.csproj @@ -48,7 +48,7 @@ - + From c6b33453cf2f80217effa37aecf6c46b8f67d01b Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Sun, 6 Sep 2026 18:46:11 +0300 Subject: [PATCH 12/43] feat: run resident adapters in the pipeline, behind one runtime interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A resident adapter could be picked from the handler, mapper, receiver or validator dropdown and could not work. The catalog lists by key prefix and never read the metadata, so residency made no difference to whether it was offered; the invoke path did not read it either and sent every packaged adapter down the classic route, where a resident one waits out the command timeout for an answer that never comes. The UI swallows the failed properties fetch, so what an operator saw was a selectable adapter with nothing to configure — not a broken one. 224 unit + 274 integration tests pass. One interface, three runtimes - IAdapterRuntime, asked in order, first to claim an adapter runs it: native (in-process), resident (rented from the pool), classic (spawned, and the catch-all, so it is asked last). IAdapterInvoker picks between them. - XchangeService, ReceivingJob and RetryAlertService now say WHAT they want run — id, role, method — with no branching on adapter kind anywhere in the pipeline. The native-versus-serverless if blocks are gone. - The role travels with the call because the id does not carry it: a mapper and a handler are both invoked as Handle, and only the role tells the native runtime which registration to resolve. Losing it would silently run the wrong adapter. - Sessions, not just calls. A receiver is Initialize, ListFiles, a GetFile and DeleteFile per item, then Finalize, and all of them have to reach ONE instance — renting per call would scatter that across pooled processes and Initialize would run somewhere the listing never sees. Catalog - SearchVersioned lists by the Kind stamped at publish time, falling back to the infolink6.s. prefix. The fallback stays because dropping it would empty the dropdown on any deployment that has not republished since the stamp existed. An adapter that declares its kind now appears whatever it is called, so reclassifying no longer means renaming — and a rename is not free, because every subscription stores the id. Tests - AdapterInvokerTests (7): routing, ordering, role and property passthrough, session release, and a session staying open until the caller closes it. Hand written fakes, no database or containers — 28ms, which is the seam this refactor was for. - ResidentPipelineAdapterTests (5): a real resident adapter answering the handler contract, the SAME instance serving three messages (a classic one is a new process per call and would answer 1 every time), a session pinned to one instance, and classic and native adapters still going through the same door. - ResidentAsPipelineAdapterTests (4): the dropdown behaviour, and both classification paths. - Mutation-verified: stopping the resident runtime claiming fails exactly the three resident tests; ignoring the stamped Kind fails exactly the metadata one. Known cost: listing by Kind reads metadata for anything the prefix did not already match. It is cached, but on a store with hundreds of adapters this is more work than the single prefixed LIST it replaces, and a registry table would be the better answer at that size. Co-Authored-By: Claude Opus 5 --- .../Resources/Adapters/SearchVersioned.cs | 76 ++++++- SW.Bitween.Api/Services/AdapterInvoker.cs | 39 ---- .../Services/Adapters/AdapterInvoker.cs | 36 ++++ .../Adapters/ClassicAdapterRuntime.cs | 42 ++++ .../Services/Adapters/IAdapterRuntime.cs | 66 +++++++ .../Services/Adapters/NativeAdapterRuntime.cs | 90 +++++++++ .../Adapters/ResidentAdapterRuntime.cs | 80 ++++++++ SW.Bitween.Api/Services/ReceivingJob.cs | 50 ++--- SW.Bitween.Api/Services/RetryAlertService.cs | 8 +- SW.Bitween.Api/Services/XchangeService.cs | 65 ++---- .../Fixtures/BitweenFixture.cs | 8 +- .../SW.Bitween.IntegrationTests.csproj | 1 + .../Tests/ResidentAsPipelineAdapterTests.cs | 171 ++++++++++++++++ .../Tests/ResidentPipelineAdapterTests.cs | 160 +++++++++++++++ SW.Bitween.SampleResidentHandler/Handler.cs | 66 +++++++ SW.Bitween.SampleResidentHandler/Program.cs | 10 + .../SW.Bitween.SampleResidentHandler.csproj | 13 ++ SW.Bitween.UnitTests/AdapterInvokerTests.cs | 186 ++++++++++++++++++ SW.Bitween.Web/Startup.cs | 8 +- SW.Bitween.sln | 15 ++ 20 files changed, 1063 insertions(+), 127 deletions(-) delete mode 100644 SW.Bitween.Api/Services/AdapterInvoker.cs create mode 100644 SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs create mode 100644 SW.Bitween.Api/Services/Adapters/ClassicAdapterRuntime.cs create mode 100644 SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs create mode 100644 SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs create mode 100644 SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/ResidentAsPipelineAdapterTests.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/ResidentPipelineAdapterTests.cs create mode 100644 SW.Bitween.SampleResidentHandler/Handler.cs create mode 100644 SW.Bitween.SampleResidentHandler/Program.cs create mode 100644 SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj create mode 100644 SW.Bitween.UnitTests/AdapterInvokerTests.cs diff --git a/SW.Bitween.Api/Resources/Adapters/SearchVersioned.cs b/SW.Bitween.Api/Resources/Adapters/SearchVersioned.cs index 5d29ae43..3f7452d9 100644 --- a/SW.Bitween.Api/Resources/Adapters/SearchVersioned.cs +++ b/SW.Bitween.Api/Resources/Adapters/SearchVersioned.cs @@ -15,11 +15,13 @@ public class SearchVersioned : IQueryHandler private readonly NativeAdapterDiscoveryService _nativeAdapterDiscovery; private readonly BitweenDbContext _dbContext; private readonly RequestContext _requestContext; + private readonly SW.Serverless.AdapterInstaller _adapterInstaller; public SearchVersioned(ServerlessOptions serverlessOptions, ICloudFilesService cloudFilesService, NativeAdapterDiscoveryService nativeAdapterDiscovery, BitweenDbContext dbContext, - RequestContext requestContext) + RequestContext requestContext, SW.Serverless.AdapterInstaller adapterInstaller) { + _adapterInstaller = adapterInstaller; _serverlessOptions = serverlessOptions; _cloudFilesService = cloudFilesService; _nativeAdapterDiscovery = nativeAdapterDiscovery; @@ -43,10 +45,16 @@ public async Task Handle(AdapterSearchRequest request) }) .ToList(); - // Get external adapters from storage - var cloudFilesList = - (await _cloudFilesService.ListAsync( - $"{_serverlessOptions.AdapterRemotePath}/infolink6.{request.Prefix}")) + // Two ways an adapter says what it is for, and both are honoured. + // + // The Kind stamped on it at publish time is the real answer: it comes from the code + // rather than from whoever typed the id, and it lets an adapter be reclassified without + // being renamed — a rename is not free, because every subscription stores the id. + // + // The infolink6.s. prefix is the old convention, and everything published before + // the stamp exists carries nothing else. Dropping it would empty this list on every + // deployment that has not republished, so it stays as the fallback. + var cloudFilesList = (await ListByKindAsync(request.Prefix)) .Where(item => item.Size > 0) .ToList(); @@ -73,5 +81,63 @@ public async Task Handle(AdapterSearchRequest request) // Return native adapters first, then external return nativeAdapters.Concat(externalAdapters); } + + /// + /// Everything published under the old naming convention for this kind, plus everything + /// that declared the kind in its metadata regardless of what it is called. + /// + private async Task> ListByKindAsync(string prefix) + { + var root = _serverlessOptions.AdapterRemotePath; + + var byConvention = (await _cloudFilesService.ListAsync($"{root}/infolink6.{prefix}")).ToList(); + var named = byConvention.Select(i => i.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); + + // The plural the UI asks with — "handlers" — against the singular an adapter declares. + var kind = prefix?.TrimEnd('s') ?? ""; + if (string.IsNullOrWhiteSpace(kind)) return byConvention; + + foreach (var item in await _cloudFilesService.ListAsync($"{root}/")) + { + if (item.Size <= 0 || named.Contains(item.Key)) continue; + + var declared = await DeclaredKindsAsync(item.Key, root); + if (declared.Contains(kind, StringComparer.OrdinalIgnoreCase)) + byConvention.Add(item); + } + + return byConvention; + } + + /// + /// The kinds one adapter declared. Metadata reads are cached by the installer, and an + /// adapter whose metadata cannot be read simply declares nothing rather than taking the + /// whole catalog down with it — the list is what an operator needs to configure anything + /// at all. + /// + private async Task DeclaredKindsAsync(string key, string root) + { + try + { + var adapterId = key.StartsWith($"{root}/", StringComparison.OrdinalIgnoreCase) + ? key[(root.Length + 1)..] + : key; + + // Versioned uploads keep the adapter id one segment up from the version. + if (Semver.IsVersionNumber(adapterId.Split('/').Last())) + adapterId = string.Join('/', adapterId.Split('/')[..^1]); + + var metadata = await _adapterInstaller.GetMetadataAsync(adapterId); + if (metadata?.AdapterValues == null) return []; + + return metadata.AdapterValues.TryGetValue("Kind", out var kinds) && !string.IsNullOrWhiteSpace(kinds) + ? kinds.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + : []; + } + catch + { + return []; + } + } } } \ No newline at end of file diff --git a/SW.Bitween.Api/Services/AdapterInvoker.cs b/SW.Bitween.Api/Services/AdapterInvoker.cs deleted file mode 100644 index f310d484..00000000 --- a/SW.Bitween.Api/Services/AdapterInvoker.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using SW.PrimitiveTypes; - -namespace SW.Bitween; - -/// -/// Runs a handler adapter, whichever kind it is: in-process for a native handler, or through -/// serverless for an uploaded one. -/// -/// -/// The choice between the two is made from the id alone and is identical wherever a handler is -/// invoked, so it lives here once. Kept as the single place that knows the adapter contract — when -/// that contract changes, a copy of this block somewhere else is what gets left behind. -/// -public class AdapterInvoker( - NativeAdapterDiscoveryService nativeAdapterDiscovery, - IServiceProvider serviceProvider) -{ - /// - /// Hands to the handler and returns whatever it produced, which is - /// null for a handler that only consumes. - /// - public async Task Handle(string handlerId, Dictionary handlerProperties, - string correlationId, XchangeFile payload) - { - if (handlerId.StartsWith(NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)) - { - var handler = nativeAdapterDiscovery.GetNativeHandler(handlerId, handlerProperties); - return await handler.Handle(payload); - } - - var serverless = serviceProvider.GetRequiredService(); - await serverless.StartAsync(handlerId, correlationId, handlerProperties); - return await serverless.InvokeAsync(nameof(IInfolinkHandler.Handle), payload); - } -} diff --git a/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs b/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs new file mode 100644 index 00000000..42a03f3a --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Adapters; + +/// +/// Picks the runtime an adapter belongs to and opens a session against it. +/// +/// The pipeline asks for a mapper, handler, validator or receiver by id and role; which of the +/// three runtimes ends up running it is decided here and nowhere else. That is what keeps the +/// choice from being copied into every call site — and what lets a test replace the whole thing. +/// +public class AdapterInvoker(IEnumerable runtimes) : IAdapterInvoker +{ + // Order matters: the classic runtime claims everything, so it has to be asked last. + private readonly IReadOnlyList _runtimes = runtimes.ToList(); + + public async Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties = null, string correlationId = null) + { + foreach (var runtime in _runtimes) + if (await runtime.CanRunAsync(adapterId)) + return await runtime.BeginAsync(adapterId, role, properties, correlationId); + + throw new BitweenException($"No runtime here knows how to run adapter '{adapterId}'."); + } + + public async Task InvokeAsync(string adapterId, AdapterRole role, string method, + object argument, IDictionary properties = null, string correlationId = null) + { + await using var session = await BeginAsync(adapterId, role, properties, correlationId); + return await session.InvokeAsync(method, argument); + } +} diff --git a/SW.Bitween.Api/Services/Adapters/ClassicAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/ClassicAdapterRuntime.cs new file mode 100644 index 00000000..8254df75 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/ClassicAdapterRuntime.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Adapters; + +/// +/// A packaged adapter spawned per invocation, answering over stdin/stdout and exiting. +/// +/// The fallback runtime: anything not native and not marked resident is run this way, which is +/// what every uploaded adapter was before residency existed. +/// +public class ClassicAdapterRuntime(IServiceProvider serviceProvider) : IAdapterRuntime +{ + /// + /// Claims everything. Registered last, so it only sees what the other runtimes declined. + /// + public Task CanRunAsync(string adapterId) => Task.FromResult(true); + + public async Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties, string correlationId) + { + var serverless = serviceProvider.GetRequiredService(); + await serverless.StartAsync(adapterId, correlationId, properties); + return new ClassicAdapterSession(serverless); + } + + private sealed class ClassicAdapterSession(IServerlessService serverless) : IAdapterSession + { + public Task InvokeAsync(string method, object argument = null) => + serverless.InvokeAsync(method, argument); + + public Task InvokeAsync(string method, object argument = null) => + serverless.InvokeAsync(method, argument); + + // The serverless service belongs to the DI scope and is disposed with it, which is how + // this path has always worked. + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs new file mode 100644 index 00000000..6d6bf468 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace SW.Bitween.Services.Adapters; + +/// +/// What an adapter is being asked to be. The id alone does not say: a mapper and a handler are +/// both invoked as Handle, and only the role distinguishes which one to resolve. +/// +public enum AdapterRole +{ + Handler, + Mapper, + Validator, + Receiver, +} + +/// +/// A run of calls against ONE adapter instance. +/// +/// A receiver is why this is a session rather than a single call: Initialize, ListFiles, a GetFile +/// and DeleteFile per item, then Finalize — all of which have to reach the same instance, or +/// Initialize runs somewhere the listing never sees. +/// +public interface IAdapterSession : IAsyncDisposable +{ + Task InvokeAsync(string method, object argument = null); + + /// For a method that returns nothing — Initialize, DeleteFile, Finalize. + Task InvokeAsync(string method, object argument = null); +} + +/// +/// One way of running an adapter. There are three, and they are genuinely different runtimes +/// rather than variations: in-process for a native adapter, a spawned process speaking +/// stdin/stdout for a classic one, and a pooled long-lived process on a socket for a resident one. +/// +/// Registered in order; the first runtime that claims an id runs it. +/// +public interface IAdapterRuntime +{ + /// + /// Whether this runtime is the one for that adapter. Async because deciding can mean reading + /// the adapter's published metadata — cached, but not free. + /// + Task CanRunAsync(string adapterId); + + Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties, string correlationId); +} + +/// +/// The one thing the pipeline calls to run an adapter. Everything that needs a mapper, handler, +/// validator or receiver goes through here, so the choice of runtime is made once and in one +/// place — and so a test can replace the lot. +/// +public interface IAdapterInvoker +{ + Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties = null, string correlationId = null); + + /// A session of exactly one call, which is what a mapper, handler or validator is. + Task InvokeAsync(string adapterId, AdapterRole role, string method, + object argument, IDictionary properties = null, string correlationId = null); +} diff --git a/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs new file mode 100644 index 00000000..56797996 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using SW.Bitween.NativeAdapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Adapters; + +/// +/// Adapters that ship inside Bitween and run in-process. No packaging, no process, no protocol — +/// the only runtime where "invoke" is a method call. +/// +public class NativeAdapterRuntime(NativeAdapterDiscoveryService discovery) : IAdapterRuntime +{ + public Task CanRunAsync(string adapterId) => + Task.FromResult(adapterId != null && adapterId.StartsWith( + NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)); + + public Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties, string correlationId) + { + var settings = properties as Dictionary + ?? new Dictionary(properties ?? new Dictionary()); + + // The role is what disambiguates: a mapper and a handler are both invoked as Handle, and + // are resolved from different registrations. + object adapter = role switch + { + AdapterRole.Mapper => discovery.GetNativeMapper(adapterId, settings), + AdapterRole.Validator => discovery.GetNativeValidator(adapterId, settings), + AdapterRole.Receiver => discovery.GetNativeReceiver(adapterId, settings), + _ => discovery.GetNativeHandler(adapterId, settings), + }; + + return Task.FromResult(new NativeAdapterSession(adapter)); + } + + /// + /// Dispatches by method name onto the resolved adapter, so the pipeline can treat a native + /// adapter exactly like a packaged one. + /// + private sealed class NativeAdapterSession(object adapter) : IAdapterSession + { + public async Task InvokeAsync(string method, object argument = null) => + (TResult)await Dispatch(method, argument); + + public Task InvokeAsync(string method, object argument = null) => Dispatch(method, argument); + + private async Task Dispatch(string method, object argument) + { + switch (adapter, method) + { + case (INativeInfolinkHandler h, nameof(IInfolinkHandler.Handle)): + return await h.Handle((XchangeFile)argument); + + case (INativeInfolinkMapper m, nameof(IInfolinkHandler.Handle)): + return await m.Handle((XchangeFile)argument); + + case (INativeInfolinkValidator v, nameof(IInfolinkValidator.Validate)): + return await v.Validate((XchangeFile)argument); + + case (INativeInfolinkReceiver r, nameof(IInfolinkReceiver.Initialize)): + await r.Initialize(); + return null; + + case (INativeInfolinkReceiver r, nameof(IInfolinkReceiver.ListFiles)): + return (await r.ListFiles()).ToList(); + + case (INativeInfolinkReceiver r, nameof(IInfolinkReceiver.GetFile)): + return await r.GetFile((string)argument); + + case (INativeInfolinkReceiver r, nameof(IInfolinkReceiver.DeleteFile)): + await r.DeleteFile((string)argument); + return null; + + case (INativeInfolinkReceiver r, nameof(IInfolinkReceiver.Finalize)): + await r.Finalize(); + return null; + + default: + throw new BitweenException( + $"Native adapter '{adapter.GetType().Name}' has nothing called '{method}'."); + } + } + + // Nothing to release: a native adapter is an object, not a process or a lease. + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs new file mode 100644 index 00000000..97d80040 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using SW.PrimitiveTypes; +using SW.Serverless; +using SW.Serverless.Resident; + +namespace SW.Bitween.Services.Adapters; + +/// +/// A packaged adapter that stays running, rented from the pool and invoked over its socket. +/// +/// The two packaged lifecycles are published identically — same zip, same key, same metadata bag — +/// and differ only in their entry point. Running a resident adapter down the classic path does not +/// fail cleanly: the host spawns it and then waits out the command timeout for an answer that +/// never comes, because the adapter dialled out and is waiting for a host that is not listening. +/// So the lifecycle has to be read before deciding how to run it. +/// +public class ResidentAdapterRuntime( + IServiceProvider serviceProvider, + ILogger logger) : IAdapterRuntime +{ + public const string LifecycleKey = "Lifecycle"; + public const string ResidentValue = "resident"; + + public async Task CanRunAsync(string adapterId) + { + if (string.IsNullOrWhiteSpace(adapterId)) return false; + + try + { + // Cached by the installer for AdapterMetadataCacheDuration, so this is not a storage + // round trip per message. + var installer = serviceProvider.GetRequiredService(); + var metadata = await installer.GetMetadataAsync(adapterId); + + return metadata?.AdapterValues != null && + metadata.AdapterValues.TryGetValue(LifecycleKey, out var lifecycle) && + string.Equals(lifecycle, ResidentValue, StringComparison.OrdinalIgnoreCase); + } + catch (Exception ex) + { + // Unreadable metadata means "not mine". The classic runtime picks it up and fails the + // way it always did, rather than this one inventing a new failure. + logger.LogDebug(ex, "Could not read the lifecycle of adapter {AdapterId}.", adapterId); + return false; + } + } + + public async Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties, string correlationId) + { + var adapters = serviceProvider.GetService() + ?? throw new BitweenException( + $"Adapter '{adapterId}' is a resident adapter, but resident adapters are not " + + "enabled on this node (Bitween:BusProvidersEnabled). It cannot run here."); + + var spec = new AdapterSpec { AdapterId = adapterId }; + foreach (var kv in properties ?? new Dictionary()) + spec.StartupValues[kv.Key] = kv.Value; + + // Rented, not started: the process is already up, so the call costs a round trip rather + // than a launch. Returning the lease is what releases it to the next message — and what + // triggers the reset that clears per-message state between borrowers. + return new ResidentAdapterSession(await adapters.RentAsync(spec)); + } + + private sealed class ResidentAdapterSession(IAdapterLease lease) : IAdapterSession + { + public Task InvokeAsync(string method, object argument = null) => + lease.InvokeAsync(method, argument); + + public Task InvokeAsync(string method, object argument = null) => + lease.InvokeAsync(method, argument); + + public ValueTask DisposeAsync() => lease.DisposeAsync(); + } +} diff --git a/SW.Bitween.Api/Services/ReceivingJob.cs b/SW.Bitween.Api/Services/ReceivingJob.cs index cad0dfab..e127a012 100644 --- a/SW.Bitween.Api/Services/ReceivingJob.cs +++ b/SW.Bitween.Api/Services/ReceivingJob.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using SW.Bitween.Services.Adapters; namespace SW.Bitween; @@ -18,7 +19,7 @@ public class ReceivingJob( BitweenDbContext dbContext, RunFlagUpdater runFlagUpdater, NativeAdapterDiscoveryService nativeAdapterDiscovery, - IServerlessService serverless, + IAdapterInvoker adapterInvoker, XchangeService xchangeService, ILogger logger) : IScheduledJob { @@ -96,41 +97,26 @@ private async Task RunReceiver( string serverlessId, IDictionary startupParameters, int subId, List createdExchangeIds) { - if (serverlessId.StartsWith(NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)) - { - var receiver = nativeAdapterDiscovery.GetNativeReceiver(serverlessId, startupParameters); - await receiver.Initialize(); - var fileList = (await receiver.ListFiles()).ToList(); + // One session for the whole run: Initialize, the listing, every GetFile and DeleteFile, and + // Finalize all have to reach the SAME instance, or Initialize runs somewhere the listing + // never sees. A resident receiver is rented from the pool and held for the duration; a + // classic one is spawned; a native one is just an object. The pipeline cannot tell. + await using var session = await adapterInvoker.BeginAsync( + serverlessId, AdapterRole.Receiver, startupParameters); - logger.LogInformation("Subscription '{SubId}' found {Count} items for retrieval.", subId, fileList.Count); + await session.InvokeAsync(nameof(IInfolinkReceiver.Initialize)); + var fileList = (await session.InvokeAsync>(nameof(IInfolinkReceiver.ListFiles))).ToList(); - foreach (var file in fileList) - { - var xchangeFile = await receiver.GetFile(file); - logger.LogInformation("Submitting received file for subscriber: '{SubId}'.", subId); - createdExchangeIds.Add(await xchangeService.SubmitSubscriptionXchange(subId, xchangeFile)); - await receiver.DeleteFile(file); - } + logger.LogInformation("Subscription '{SubId}' found {Count} items for retrieval.", subId, fileList.Count); - await receiver.Finalize(); - } - else + foreach (var file in fileList) { - await serverless.StartAsync(serverlessId, null, startupParameters); - await serverless.InvokeAsync(nameof(IInfolinkReceiver.Initialize), null); - var fileList = (await serverless.InvokeAsync>(nameof(IInfolinkReceiver.ListFiles), null)).ToList(); - - logger.LogInformation("Subscription '{SubId}' found {Count} items for retrieval.", subId, fileList.Count); - - foreach (var file in fileList) - { - var xchangeFile = await serverless.InvokeAsync(nameof(IInfolinkReceiver.GetFile), file); - logger.LogInformation("Submitting received file for subscriber: '{SubId}'.", subId); - createdExchangeIds.Add(await xchangeService.SubmitSubscriptionXchange(subId, xchangeFile)); - await serverless.InvokeAsync(nameof(IInfolinkReceiver.DeleteFile), file); - } - - await serverless.InvokeAsync(nameof(IInfolinkReceiver.Finalize), null); + var xchangeFile = await session.InvokeAsync(nameof(IInfolinkReceiver.GetFile), file); + logger.LogInformation("Submitting received file for subscriber: '{SubId}'.", subId); + createdExchangeIds.Add(await xchangeService.SubmitSubscriptionXchange(subId, xchangeFile)); + await session.InvokeAsync(nameof(IInfolinkReceiver.DeleteFile), file); } + + await session.InvokeAsync(nameof(IInfolinkReceiver.Finalize)); } } diff --git a/SW.Bitween.Api/Services/RetryAlertService.cs b/SW.Bitween.Api/Services/RetryAlertService.cs index 414e3be1..3721435b 100644 --- a/SW.Bitween.Api/Services/RetryAlertService.cs +++ b/SW.Bitween.Api/Services/RetryAlertService.cs @@ -8,6 +8,7 @@ using SW.Bitween.Domain; using SW.Bitween.Model; using SW.PrimitiveTypes; +using SW.Bitween.Services.Adapters; namespace SW.Bitween; @@ -22,7 +23,7 @@ namespace SW.Bitween; /// public class RetryAlertService( BitweenDbContext dbContext, - AdapterInvoker adapterInvoker, + IAdapterInvoker adapterInvoker, ILogger logger) : IConsume { public async Task Process(RetryBudgetExhaustedEvent message) @@ -113,8 +114,9 @@ private async Task Send(RetryAlertTarget target, RetryBudgetExhaustedNotificatio try { - await adapterInvoker.Handle(target.HandlerId, handlerProperties, - notification.CorrelationId ?? xchangeId, payload); + await adapterInvoker.InvokeAsync( + target.HandlerId, AdapterRole.Handler, nameof(IInfolinkHandler.Handle), payload, + handlerProperties, notification.CorrelationId ?? xchangeId); dbContext.Add(XchangeNotification.ForRetryBudgetAlert(xchangeId)); } diff --git a/SW.Bitween.Api/Services/XchangeService.cs b/SW.Bitween.Api/Services/XchangeService.cs index 3f7260ca..6ea7f5e4 100644 --- a/SW.Bitween.Api/Services/XchangeService.cs +++ b/SW.Bitween.Api/Services/XchangeService.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using SW.Bitween.Services.Adapters; using SW.Bus.RabbitMqExtensions; namespace SW.Bitween; @@ -35,13 +36,13 @@ public class XchangeService : private readonly ILogger _logger; private readonly IInfolinkCache _BitweenCache; private readonly NativeAdapterDiscoveryService _nativeAdapterDiscovery; - private readonly AdapterInvoker _adapterInvoker; + private readonly IAdapterInvoker _adapterInvoker; public XchangeService(BitweenOptions BitweenSettings, BitweenDbContext dbContext, FilterService filterService, ICloudFilesService cloudFiles, IServiceProvider serviceProvider, IPublish publish, ILogger logger, IInfolinkCache BitweenCache, - NativeAdapterDiscoveryService nativeAdapterDiscovery, AdapterInvoker adapterInvoker) + NativeAdapterDiscoveryService nativeAdapterDiscovery, IAdapterInvoker adapterInvoker) { _adapterInvoker = adapterInvoker; _BitweenSettings = BitweenSettings; @@ -262,18 +263,12 @@ private async Task RunMapper(Xchange xchange, XchangeFile xchangeFi mapperProperties["xchangeid"] = xchange.Id; // Check if it's a native adapter - if (xchange.MapperId.StartsWith(NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)) - { - var handler = _nativeAdapterDiscovery.GetNativeMapper(xchange.MapperId, mapperProperties); - xchangeFile = await handler.Handle(xchangeFile); - } - else - { - // Use serverless for external adapters - var serverless = _serviceProvider.GetRequiredService(); - await serverless.StartAsync(xchange.MapperId, xchange.CorrelationId ?? xchange.Id, mapperProperties); - xchangeFile = await serverless.InvokeAsync(nameof(IInfolinkHandler.Handle), xchangeFile); - } + // No branching on the adapter's kind: the invoker decides which of the three runtimes + // owns this id — in-process, spawned, or a rented resident instance — and the pipeline + // only says what it wants run. + xchangeFile = await _adapterInvoker.InvokeAsync( + xchange.MapperId, AdapterRole.Mapper, nameof(IInfolinkHandler.Handle), xchangeFile, + mapperProperties, xchange.CorrelationId ?? xchange.Id); if (xchangeFile is null) throw new BitweenException( @@ -288,23 +283,9 @@ public async Task RunValidator(string validatorId, IDictionary p { if (validatorId == null) return; - InfolinkValidatorResult result; - - // Check if it's a native adapter - if (validatorId.StartsWith(NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)) - { - var validator = _nativeAdapterDiscovery.GetNativeValidator(validatorId, properties); - - result = await validator.Validate(xchangeFile); - } - else - { - // Use serverless for external adapters - var serverless = _serviceProvider.GetRequiredService(); - await serverless.StartAsync(validatorId, null, properties); - result = await serverless.InvokeAsync(nameof(IInfolinkValidator.Validate), - xchangeFile); - } + var result = await _adapterInvoker.InvokeAsync( + validatorId, AdapterRole.Validator, nameof(IInfolinkValidator.Validate), + xchangeFile, properties); if (!result.Success) throw new SWValidationException(result.Validations); @@ -317,19 +298,9 @@ private async Task RunHandler(Xchange xchange, XchangeFile xchangeF var handlerProperties = xchange.HandlerProperties.ToDictionary(); handlerProperties["xchangeid"] = xchange.Id; - // Check if it's a native adapter - if (xchange.HandlerId.StartsWith(NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)) - { - var handler = _nativeAdapterDiscovery.GetNativeHandler(xchange.HandlerId, handlerProperties); - xchangeFile = await handler.Handle(xchangeFile); - } - else - { - // Use serverless for external adapters - var serverless = _serviceProvider.GetRequiredService(); - await serverless.StartAsync(xchange.HandlerId, xchange.CorrelationId ?? xchange.Id, handlerProperties); - xchangeFile = await serverless.InvokeAsync(nameof(IInfolinkHandler.Handle), xchangeFile); - } + xchangeFile = await _adapterInvoker.InvokeAsync( + xchange.HandlerId, AdapterRole.Handler, nameof(IInfolinkHandler.Handle), xchangeFile, + handlerProperties, xchange.CorrelationId ?? xchange.Id); if (xchangeFile != null) await AddFile(xchange.Id, XchangeFileType.Response, xchangeFile); @@ -744,8 +715,10 @@ private async Task NotifyResult(Notifier notifier, XchangeResult xchangeResult, try { - await _adapterInvoker.Handle(notifier.HandlerId, handlerProperties, correlationId, - new XchangeFile(JsonConvert.SerializeObject(notificationData), xchangeResult.Id)); + await _adapterInvoker.InvokeAsync( + notifier.HandlerId, AdapterRole.Handler, nameof(IInfolinkHandler.Handle), + new XchangeFile(JsonConvert.SerializeObject(notificationData), xchangeResult.Id), + handlerProperties, correlationId); _dbContext.Add(new XchangeNotification(xchangeResult.Id, notifier.Id, notifier.Name)); } diff --git a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs index 448587ec..8b0f8838 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs @@ -28,6 +28,7 @@ using Testcontainers.PostgreSql; using Testcontainers.RabbitMq; using Xunit; +using SW.Bitween.Services.Adapters; namespace SW.Bitween.IntegrationTests.Fixtures; @@ -251,7 +252,12 @@ await Task.WhenAll(_postgres.StartAsync(), _rabbitMq.StartAsync(), _mailHog.Star services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + // Registration ORDER is the routing order: each runtime is asked whether an + // adapter is its own, and the classic one claims everything, so it must be asked last. + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index f2794a01..fa2aa858 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -50,6 +50,7 @@ + diff --git a/SW.Bitween.IntegrationTests/Tests/ResidentAsPipelineAdapterTests.cs b/SW.Bitween.IntegrationTests/Tests/ResidentAsPipelineAdapterTests.cs new file mode 100644 index 00000000..de880f25 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/ResidentAsPipelineAdapterTests.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Newtonsoft.Json; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// A resident adapter used where a CLASSIC one is expected — as a subscription's handler, mapper, +/// receiver or validator. +/// +/// The two lifecycles are installed identically: same zip, same cloud key, same metadata bag. What +/// differs is the entry point — Runner.Run speaks the classic stdin/stdout protocol, while +/// Runner.RunResident dials back over a socket and waits for the resident host. Nothing in the +/// adapter catalog or the invoke path looks at which one an adapter uses, so the question these +/// tests answer is what an operator actually gets when they pick one from the dropdown. +/// +[Collection("Bitween")] +public class ResidentAsPipelineAdapterTests +{ + // The catalog lists by key prefix — infolink6.{handlers|mappers|receivers|validators} — so an + // adapter has to be installed under one of those to be selectable at all. + private const string ResidentHandlerId = "infolink6.handlers.residenttest"; + private const string ClassicHandlerId = "infolink6.handlers.classictest"; + + private readonly BitweenFixture _fixture; + + public ResidentAsPipelineAdapterTests(BitweenFixture fixture) => _fixture = fixture; + + private async Task InstallAsync() + { + await using var scope = _fixture.CreateScope(); + var cloudFiles = scope.ServiceProvider.GetRequiredService(); + + // The same sample twice, under two names. The only difference the host could possibly see + // is the metadata, which is the point. + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.SampleHandler", ClassicHandlerId, "SW.Bitween.SampleHandler.dll"); + + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.Adapters.Bus.RabbitMq", ResidentHandlerId, + "SW.Bitween.Adapters.Bus.RabbitMq.dll", + new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); + } + + /// + /// Both show up. The catalog groups cloud keys by name and never reads the metadata, so + /// residency makes no difference to whether an operator can pick it. + /// + [Fact] + public async Task A_resident_adapter_appears_in_the_handler_dropdown() + { + await InstallAsync(); + + var listed = await ListAsync("handlers"); + + Assert.Contains(ClassicHandlerId, listed); + Assert.Contains(ResidentHandlerId, listed); + } + + /// + /// And here is the problem. The dropdown offers it, and asking for its startup values — which + /// is how the UI builds the properties form — goes down the classic path: spawn the process, + /// talk stdin/stdout, wait. A resident adapter is not listening on stdin; it dialled out and is + /// waiting for a host that, on this path, never calls. + /// + /// The UI swallows this and shows an adapter with no properties, so what an operator sees is a + /// selectable adapter that simply has nothing to configure — not a broken one. + /// + [Fact] + public async Task Asking_a_resident_adapter_for_its_startup_values_does_not_work() + { + await InstallAsync(); + + // The classic one answers. + var classic = await StartupValuesAsync(ClassicHandlerId); + Assert.NotNull(classic); + + // The resident one does not — it either fails outright or hands back nothing, and either + // way the properties form is empty. + var resident = await Record.ExceptionAsync(() => StartupValuesAsync(ResidentHandlerId)); + + if (resident == null) + { + var values = await StartupValuesAsync(ResidentHandlerId); + Assert.True(values is not { Count: > 0 }, + "a resident adapter answered the classic startup-values handshake, which would mean " + + "the two lifecycles are interchangeable after all"); + } + } + + /// + /// An adapter named nothing like the convention still appears, because it SAID what it is. + /// + /// This is the point of stamping the kind at publish time: the id stops having to carry the + /// classification, so an adapter can be reclassified without being renamed — and a rename is + /// not free, because every subscription stores the id it was configured with. + /// + [Fact] + public async Task An_adapter_that_declares_its_kind_appears_whatever_it_is_called() + { + const string oddlyNamedId = "acme.orders.processor"; + + await using (var scope = _fixture.CreateScope()) + { + var cloudFiles = scope.ServiceProvider.GetRequiredService(); + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.SampleHandler", oddlyNamedId, "SW.Bitween.SampleHandler.dll", + new Dictionary { ["Kind"] = "handler" }); + } + + var handlers = await ListAsync("handlers"); + Assert.Contains(oddlyNamedId, handlers); + + // And it is not offered as something it never claimed to be. + var validators = await ListAsync("validators"); + Assert.DoesNotContain(oddlyNamedId, validators); + } + + /// + /// The old convention still works, or the catalog would empty itself on any deployment whose + /// adapters have not been republished since the stamp existed. + /// + [Fact] + public async Task An_adapter_with_no_declared_kind_is_still_found_by_its_name() + { + await InstallAsync(); + + var listed = await ListAsync("handlers"); + + // Installed with no Kind at all — found purely by the infolink6.handlers. prefix. + Assert.Contains(ClassicHandlerId, listed); + } + + // ---------------------------------------------------------------- helpers + + private async Task> ListAsync(string prefix) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var result = await handler.Handle(new AdapterSearchRequest { Prefix = prefix }); + + // The handler returns an anonymous-typed sequence; go through JSON rather than reflect. + var json = JsonConvert.SerializeObject(result); + var rows = JsonConvert.DeserializeObject>>(json) ?? new(); + + return rows + .Select(r => r.TryGetValue("Key", out var k) ? k?.ToString() : null) + .Where(k => !string.IsNullOrEmpty(k)) + .Select(k => k!) + .ToList(); + } + + private async Task?> StartupValuesAsync(string adapterId) + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var result = await handler.Handle(adapterId); + return result as IDictionary; + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/ResidentPipelineAdapterTests.cs b/SW.Bitween.IntegrationTests/Tests/ResidentPipelineAdapterTests.cs new file mode 100644 index 00000000..88e180ed --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/ResidentPipelineAdapterTests.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// A RESIDENT adapter used as an ordinary pipeline adapter — a subscription's handler or mapper. +/// +/// The two packaged lifecycles are published identically and differ only in their entry point, so +/// nothing about the pipeline's contract changes. What used to happen is that the invoke path did +/// not look at which one an adapter used and sent every packaged adapter down the classic route, +/// where a resident one waits out the command timeout for an answer that never comes. +/// +/// These run the real thing: a resident adapter installed from cloud storage, started as a process, +/// and invoked through the same code path a classic adapter goes through. +/// +[Collection("Bitween")] +public class ResidentPipelineAdapterTests +{ + private const string ResidentHandlerId = "infolink6.handlers.residentsample"; + + private readonly BitweenFixture _fixture; + + public ResidentPipelineAdapterTests(BitweenFixture fixture) => _fixture = fixture; + + private async Task InstallAsync() + { + await using var scope = _fixture.CreateScope(); + var cloudFiles = scope.ServiceProvider.GetRequiredService(); + + await AdapterInstaller.InstallAsync(cloudFiles, + "SW.Bitween.SampleResidentHandler", ResidentHandlerId, + "SW.Bitween.SampleResidentHandler.dll", + new Dictionary { ["Protocol"] = "2", ["Lifecycle"] = "resident" }); + } + + /// + /// The whole point: a resident adapter answers the ordinary handler contract, through the + /// ordinary invoke path, with nothing about the call saying which lifecycle it is. + /// + [Fact] + public async Task A_resident_adapter_answers_the_handler_contract() + { + await InstallAsync(); + + await using var scope = _fixture.CreateScope(); + var invoker = scope.ServiceProvider.GetRequiredService(); + + var result = await invoker.InvokeAsync( + ResidentHandlerId, AdapterRole.Handler, nameof(IInfolinkHandler.Handle), + new XchangeFile("""{"hello":"world"}""", "in.json"), + new Dictionary { ["Greeting"] = "from-a-resident" }); + + Assert.NotNull(result); + + var body = JObject.Parse(result!.Data); + Assert.Equal("from-a-resident", body.Value("greeting")); + Assert.Equal("world", body["echo"]?.Value("hello")); + } + + /// + /// The distinction a single call cannot show: the SAME process serves more than one message. + /// + /// A classic adapter is a new process per invocation and would answer 1 every time, so a count + /// above 1 is the proof that the instance was kept and reused — which is the only reason to + /// make an adapter resident in the first place. + /// + [Fact] + public async Task The_same_instance_serves_more_than_one_message() + { + await InstallAsync(); + + await using var scope = _fixture.CreateScope(); + var invoker = scope.ServiceProvider.GetRequiredService(); + + var counts = new List(); + for (var i = 0; i < 3; i++) + { + var result = await invoker.InvokeAsync( + ResidentHandlerId, AdapterRole.Handler, nameof(IInfolinkHandler.Handle), + new XchangeFile($$"""{"n":{{i}}}""", "in.json")); + + counts.Add(JObject.Parse(result!.Data).Value("handled")); + } + + Assert.Contains(counts, c => c > 1); + Assert.Equal(counts.OrderBy(c => c).ToList(), counts); + } + + /// + /// A session holds one instance across several calls, which is what a receiver depends on — + /// Initialize, the listing and every GetFile have to reach the same place. + /// + [Fact] + public async Task A_session_keeps_every_call_on_one_instance() + { + await InstallAsync(); + + await using var scope = _fixture.CreateScope(); + var invoker = scope.ServiceProvider.GetRequiredService(); + + await using var session = await invoker.BeginAsync(ResidentHandlerId, AdapterRole.Handler); + + var first = await session.InvokeAsync( + nameof(IInfolinkHandler.Handle), new XchangeFile("""{"n":1}""", "a.json")); + var second = await session.InvokeAsync( + nameof(IInfolinkHandler.Handle), new XchangeFile("""{"n":2}""", "b.json")); + + var firstCount = JObject.Parse(first!.Data).Value("handled"); + var secondCount = JObject.Parse(second!.Data).Value("handled"); + + Assert.Equal(firstCount + 1, secondCount); + } + + /// + /// And a classic adapter still goes the classic way. The routing has to keep the existing + /// lifecycle working, or fixing one kind would have broken every adapter already deployed. + /// + [Fact] + public async Task A_classic_adapter_still_runs_through_the_same_invoker() + { + await using var scope = _fixture.CreateScope(); + var invoker = scope.ServiceProvider.GetRequiredService(); + + var result = await invoker.InvokeAsync( + "sw.bitween.samplehandler", AdapterRole.Handler, nameof(IInfolinkHandler.Handle), + new XchangeFile("hello", "in.txt"), + new Dictionary { ["ContentType"] = "text/plain" }); + + Assert.NotNull(result); + Assert.Equal("hello", result!.Data); + } + + /// A native adapter goes through the same door too, resolved in-process. + [Fact] + public async Task A_native_adapter_runs_through_the_same_invoker() + { + await using var scope = _fixture.CreateScope(); + var invoker = scope.ServiceProvider.GetRequiredService(); + + // Resolving it is the assertion: the native runtime claims the id and returns a session + // rather than the classic one trying to find a package under that name. + await using var session = await invoker.BeginAsync( + "native.smtp", AdapterRole.Handler, + new Dictionary { ["Host"] = "localhost" }); + + Assert.NotNull(session); + } +} diff --git a/SW.Bitween.SampleResidentHandler/Handler.cs b/SW.Bitween.SampleResidentHandler/Handler.cs new file mode 100644 index 00000000..d3f49237 --- /dev/null +++ b/SW.Bitween.SampleResidentHandler/Handler.cs @@ -0,0 +1,66 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using SW.PrimitiveTypes; +using SW.Serverless.Sdk.Resident; + +namespace SW.Bitween.SampleResidentHandler; + +/// +/// A handler that STAYS RUNNING, used exactly where a classic one would be — as a subscription's +/// handler or mapper. +/// +/// It implements IInfolinkHandler like any other, so nothing about the pipeline's contract changes. +/// What differs is that the process is started once and kept, which is what a real one would use to +/// hold an open connection or a warm cache instead of paying for them on every message. +/// +/// It counts what it has handled and reports that on the heartbeat, so a test can prove the SAME +/// process served more than one message — which is the whole distinction from the classic +/// lifecycle, and not something a single call could show. +/// +public class Handler : IResidentAdapter, IInfolinkHandler +{ + private IAdapterContext _context; + private int _handled; + private string _greeting = "resident"; + + public Task StartAsync(IAdapterContext context, CancellationToken cancellationToken) + { + _context = context; + _greeting = context.StartupValueOf("Greeting") ?? "resident"; + context.LogInformation($"Resident handler ready, greeting '{_greeting}'."); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public Task GetStatusAsync() + { + var status = new AdapterStatus { Connected = true, State = "Ready" }; + status.Details["handled"] = _handled.ToString(); + return Task.FromResult(status); + } + + /// + /// The ordinary handler contract. The pipeline calls this by name and cannot tell which + /// lifecycle answered it. + /// + public Task Handle(XchangeFile xchangeFile) + { + var count = Interlocked.Increment(ref _handled); + var body = xchangeFile?.Data ?? ""; + + // The count is in the output on purpose: a classic adapter is a new process per message + // and would answer 1 every time, so anything above 1 proves this instance was reused. + var output = $"{{\"greeting\":\"{_greeting}\",\"handled\":{count},\"echo\":{Body(body)}}}"; + + _context?.Metric("resident.handler.handled", 1); + return Task.FromResult(new XchangeFile(output, xchangeFile?.Filename)); + } + + /// Keeps the echoed payload valid JSON whether or not the input was. + private static string Body(string body) => + string.IsNullOrWhiteSpace(body) ? "null" + : body.TrimStart().StartsWith('{') || body.TrimStart().StartsWith('[') ? body + : "\"" + body.Replace("\\", "\\\\").Replace("\"", "\\\"") + "\""; +} diff --git a/SW.Bitween.SampleResidentHandler/Program.cs b/SW.Bitween.SampleResidentHandler/Program.cs new file mode 100644 index 00000000..504e260b --- /dev/null +++ b/SW.Bitween.SampleResidentHandler/Program.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Bitween.SampleResidentHandler; + +static class Program +{ + // The only line that differs from the classic sample handler. + static Task Main() => Runner.RunResident(new Handler()); +} diff --git a/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj b/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj new file mode 100644 index 00000000..4c725931 --- /dev/null +++ b/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj @@ -0,0 +1,13 @@ + + + + Exe + net8.0 + SW.Bitween.SampleResidentHandler + + + + + + + diff --git a/SW.Bitween.UnitTests/AdapterInvokerTests.cs b/SW.Bitween.UnitTests/AdapterInvokerTests.cs new file mode 100644 index 00000000..52bc839c --- /dev/null +++ b/SW.Bitween.UnitTests/AdapterInvokerTests.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.UnitTests; + +/// +/// Which runtime runs an adapter. +/// +/// There are three — in-process, a spawned process, a pooled long-lived one — and they are not +/// interchangeable: sending a resident adapter down the classic path does not fail cleanly, it +/// waits out the command timeout for an answer that will never come. Choosing correctly is +/// therefore not a detail, and it is made here rather than at every call site so there is one +/// place to get it right and one place to test. +/// +/// No database, no processes, no containers: the whole point of the seam is that this is testable +/// with a fake. +/// +[TestClass] +public class AdapterInvokerTests +{ + [TestMethod] + public async Task The_first_runtime_that_claims_an_adapter_runs_it() + { + var native = new FakeRuntime("native", claims: id => id.StartsWith("native.")); + var resident = new FakeRuntime("resident", claims: id => id.StartsWith("res.")); + var classic = new FakeRuntime("classic", claims: _ => true); + + var invoker = new AdapterInvoker([native, resident, classic]); + + await invoker.InvokeAsync("res.thing", AdapterRole.Handler, "Handle", null); + + Assert.AreEqual(1, resident.Begins); + Assert.AreEqual(0, native.Begins); + Assert.AreEqual(0, classic.Begins, "a later runtime ran one an earlier one had claimed"); + } + + /// + /// The classic runtime claims everything, so registration order is the routing. If it were + /// asked first it would swallow every adapter, including the ones it cannot run. + /// + [TestMethod] + public async Task The_catch_all_runtime_only_sees_what_the_others_declined() + { + var resident = new FakeRuntime("resident", claims: id => id.StartsWith("res.")); + var classic = new FakeRuntime("classic", claims: _ => true); + + var invoker = new AdapterInvoker([resident, classic]); + + await invoker.InvokeAsync("something.else", AdapterRole.Mapper, "Handle", null); + + Assert.AreEqual(1, classic.Begins); + Assert.AreEqual(0, resident.Begins); + } + + /// + /// The role travels with the call. A mapper and a handler are both invoked as Handle, and only + /// the role tells a native runtime which of the two registrations to resolve — so losing it + /// here would silently run the wrong adapter. + /// + [TestMethod] + public async Task The_role_reaches_the_runtime() + { + var runtime = new FakeRuntime("only", claims: _ => true); + var invoker = new AdapterInvoker([runtime]); + + await invoker.InvokeAsync("x", AdapterRole.Validator, "Validate", null); + + Assert.AreEqual(AdapterRole.Validator, runtime.LastRole); + } + + [TestMethod] + public async Task Properties_and_the_correlation_id_reach_the_runtime() + { + var runtime = new FakeRuntime("only", claims: _ => true); + var invoker = new AdapterInvoker([runtime]); + + var properties = new Dictionary { ["Host"] = "broker" }; + await invoker.InvokeAsync("x", AdapterRole.Handler, "Handle", null, properties, "corr-1"); + + Assert.AreEqual("broker", runtime.LastProperties?["Host"]); + Assert.AreEqual("corr-1", runtime.LastCorrelationId); + } + + /// + /// A one-call invoke has to release the session. For a resident adapter the session IS a + /// pooled lease, so leaking one takes a warm instance out of circulation for good — and the + /// pool runs dry with no error to say why. + /// + [TestMethod] + public async Task A_single_call_still_releases_the_session() + { + var runtime = new FakeRuntime("only", claims: _ => true); + var invoker = new AdapterInvoker([runtime]); + + await invoker.InvokeAsync("x", AdapterRole.Handler, "Handle", null); + + Assert.IsTrue(runtime.LastSession.Disposed, "the session was not released"); + } + + /// A session held by the caller is theirs to release, and is not disposed early. + [TestMethod] + public async Task A_session_stays_open_until_the_caller_disposes_it() + { + var runtime = new FakeRuntime("only", claims: _ => true); + var invoker = new AdapterInvoker([runtime]); + + var session = await invoker.BeginAsync("x", AdapterRole.Receiver); + await session.InvokeAsync("Initialize"); + await session.InvokeAsync>("ListFiles"); + + Assert.IsFalse(runtime.LastSession.Disposed); + + await session.DisposeAsync(); + Assert.IsTrue(runtime.LastSession.Disposed); + + // Every call in the session went to ONE instance, which is what a receiver depends on. + Assert.AreEqual(1, runtime.Begins); + CollectionAssert.AreEqual(new[] { "Initialize", "ListFiles" }, runtime.LastSession.Calls); + } + + [TestMethod] + public async Task An_adapter_no_runtime_claims_is_a_clear_error() + { + var invoker = new AdapterInvoker([new FakeRuntime("picky", claims: _ => false)]); + + var error = await Assert.ThrowsExceptionAsync( + () => invoker.InvokeAsync("orphan", AdapterRole.Handler, "Handle", null)); + + StringAssert.Contains(error.Message, "orphan"); + } + + // ---------------------------------------------------------------- fakes + + private sealed class FakeRuntime(string name, Func claims) : IAdapterRuntime + { + public int Begins { get; private set; } + public AdapterRole LastRole { get; private set; } + public IDictionary LastProperties { get; private set; } + public string LastCorrelationId { get; private set; } + public FakeSession LastSession { get; private set; } + + public override string ToString() => name; + + public Task CanRunAsync(string adapterId) => Task.FromResult(claims(adapterId)); + + public Task BeginAsync(string adapterId, AdapterRole role, + IDictionary properties, string correlationId) + { + Begins++; + LastRole = role; + LastProperties = properties; + LastCorrelationId = correlationId; + LastSession = new FakeSession(); + return Task.FromResult(LastSession); + } + } + + private sealed class FakeSession : IAdapterSession + { + public List Calls { get; } = []; + public bool Disposed { get; private set; } + + public Task InvokeAsync(string method, object argument = null) + { + Calls.Add(method); + return Task.FromResult(default(TResult)); + } + + public Task InvokeAsync(string method, object argument = null) + { + Calls.Add(method); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Disposed = true; + return ValueTask.CompletedTask; + } + } +} diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index f666eb32..86a1280d 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -45,6 +45,7 @@ using SW.Scheduler.SqlServer; using SqlAuthenticationProvider = Microsoft.Data.SqlClient.SqlAuthenticationProvider; using SqlAuthenticationMethod = Microsoft.Data.SqlClient.SqlAuthenticationMethod; +using SW.Bitween.Services.Adapters; namespace SW.Bitween.Web { @@ -80,7 +81,12 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + // Registration ORDER is the routing order: each runtime is asked whether an + // adapter is its own, and the classic one claims everything, so it must be asked last. + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/SW.Bitween.sln b/SW.Bitween.sln index 9dd1d570..1df99d77 100644 --- a/SW.Bitween.sln +++ b/SW.Bitween.sln @@ -37,6 +37,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Bus.Rab EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.Adapters.Bus.Sqs", "SW.Bitween.Adapters.Bus.Sqs\SW.Bitween.Adapters.Bus.Sqs.csproj", "{1125FB09-88E3-405B-80C0-62C73B1AB9F8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Bitween.SampleResidentHandler", "SW.Bitween.SampleResidentHandler\SW.Bitween.SampleResidentHandler.csproj", "{4BA1230D-F572-45FA-983A-BE5BEBE380DB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -215,6 +217,18 @@ Global {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x64.Build.0 = Release|Any CPU {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x86.ActiveCfg = Release|Any CPU {1125FB09-88E3-405B-80C0-62C73B1AB9F8}.Release|x86.Build.0 = Release|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Debug|x64.ActiveCfg = Debug|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Debug|x64.Build.0 = Debug|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Debug|x86.ActiveCfg = Debug|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Debug|x86.Build.0 = Debug|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Release|Any CPU.Build.0 = Release|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Release|x64.ActiveCfg = Release|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Release|x64.Build.0 = Release|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Release|x86.ActiveCfg = Release|Any CPU + {4BA1230D-F572-45FA-983A-BE5BEBE380DB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -229,6 +243,7 @@ Global {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} = {5F58DD63-8ABF-4148-A594-0D9881F39142} {5E93D24F-EA1A-4788-B19F-326215B9CD2B} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} {1125FB09-88E3-405B-80C0-62C73B1AB9F8} = {BE50F904-02D2-E9B7-ADFA-F9CD0747F0AC} + {4BA1230D-F572-45FA-983A-BE5BEBE380DB} = {5F58DD63-8ABF-4148-A594-0D9881F39142} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {F314D530-ADF8-43EC-A747-26CA23CCC4F7} From 4d869f9aa9b570ca6150857ae6a970b3ea1d619e Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Mon, 7 Sep 2026 17:48:46 +0300 Subject: [PATCH 13/43] chore: SimplyWorks.Serverless 8.1.19, and declare the bus adapters as bus adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the ten references from 8.1.16 and stamps [AdapterKind("bus")] on both bus handlers. The attribute did not exist in 8.1.16 — it ships with the installer work merged as simplify9/SW-Serverless#117. The installer now reads it at publish time and writes Kind and Lifecycle onto the package, which is the only way a host can find a provider it did not publish itself: Bitween's own are found by their "bitween." id prefix, and a prefix cannot possibly match a third party's adapter. Renaming an adapter to carry its role in the id is not an option either, since hosts store the id against every configuration that uses it. Bitween's two adapters carry the attribute so they are discoverable both ways and the prefix can eventually retire. Verified with the installer's own AdapterDescriber over both published packages: Lifecycle: resident, Kind: 'bus'. Co-Authored-By: Claude Opus 5 --- SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs | 2 ++ .../SW.Bitween.Adapters.Bus.RabbitMq.csproj | 2 +- SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj | 2 +- SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs | 2 ++ SW.Bitween.Api/SW.Bitween.Api.csproj | 2 +- SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj | 2 +- .../SW.Bitween.SampleConfigurableAdapter.csproj | 2 +- SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj | 2 +- SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj | 2 +- .../SW.Bitween.SampleResidentHandler.csproj | 2 +- SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj | 2 +- SW.Bitween.Web/SW.Bitween.Web.csproj | 2 +- 12 files changed, 14 insertions(+), 10 deletions(-) diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs index dd01267b..680cf832 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Options; using RabbitMQ.Client; using RabbitMQ.Client.Events; +using SW.Serverless.Sdk; using SW.Serverless.Sdk.Resident; using System; using System.Collections.Generic; @@ -23,6 +24,7 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; /// A host rejection becomes BasicNack(requeue: true), so a Bitween outage does not lose the /// customer's messages — it just stops draining their queue, which is the correct failure. /// +[AdapterKind("bus")] public class RabbitBusHandler : IResidentAdapter { private readonly RabbitOptions _options; diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj index 6d02ff56..e0f27836 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj +++ b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj @@ -9,6 +9,6 @@ - + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj index af659718..e906aa8d 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj +++ b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj @@ -7,6 +7,6 @@ - + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs index dbda538f..ede71f6f 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json.Linq; +using SW.Serverless.Sdk; using SW.Serverless.Sdk.Resident; using System; using System.Collections.Generic; @@ -36,6 +37,7 @@ namespace SW.Bitween.Adapters.Bus.Sqs; /// envelope they arrive in. The SP-API request/response calls themselves are ordinary HTTPS and /// belong in a mapper or handler, not here. /// +[AdapterKind("bus")] public class SqsBusHandler : IResidentAdapter { private readonly SqsOptions _options; diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj index 16fdb33b..eb729916 100644 --- a/SW.Bitween.Api/SW.Bitween.Api.csproj +++ b/SW.Bitween.Api/SW.Bitween.Api.csproj @@ -6,7 +6,7 @@ - + diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index fa2aa858..023ef673 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -31,7 +31,7 @@ - + diff --git a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj index d93417b5..230ce755 100644 --- a/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj +++ b/SW.Bitween.SampleConfigurableAdapter/SW.Bitween.SampleConfigurableAdapter.csproj @@ -5,6 +5,6 @@ SW.Bitween.SampleConfigurableAdapter - + diff --git a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj index f51ac2d1..8762cf52 100644 --- a/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj +++ b/SW.Bitween.SampleHandler/SW.Bitween.SampleHandler.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj index 165bed0d..26b65f24 100644 --- a/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj +++ b/SW.Bitween.SampleMapper/SW.Bitween.SampleMapper.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj b/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj index 4c725931..98ea66ac 100644 --- a/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj +++ b/SW.Bitween.SampleResidentHandler/SW.Bitween.SampleResidentHandler.csproj @@ -7,7 +7,7 @@ - + diff --git a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj index 76436d7b..7ceb6db5 100644 --- a/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj +++ b/SW.Bitween.SampleValidator/SW.Bitween.SampleValidator.csproj @@ -8,7 +8,7 @@ - + diff --git a/SW.Bitween.Web/SW.Bitween.Web.csproj b/SW.Bitween.Web/SW.Bitween.Web.csproj index 18562abd..d8907143 100644 --- a/SW.Bitween.Web/SW.Bitween.Web.csproj +++ b/SW.Bitween.Web/SW.Bitween.Web.csproj @@ -48,7 +48,7 @@ - + From cfe71b451c7e64d538370b43f179c0f4c59547a2 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Mon, 7 Sep 2026 17:48:59 +0300 Subject: [PATCH 14/43] feat: a failed connection says what the adapter said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failing data source reported "Adapter stream closed." — true, useless, and the same sentence for a wrong password, a wrong port and an unreachable host. The adapter's own exception went to stderr and nowhere else, and the row's LastException stayed null, because that field only ever held what a LIVE adapter reported about itself. AdapterFailureReader undoes the SDK's wire encoding, drops the capture timestamp and the stack frames, and keeps the exception's first line plus the innermost "--->". That last part is what makes it a diagnosis rather than a shrug: "None of the specified endpoints were reachable" is equally true of a wrong host, a wrong port or a firewall, while "Cannot determine the frame size" says TLS was spoken at a plaintext port and nothing else does. The adapter's reason leads and Bitween's own observation goes in brackets, so the useful half survives truncation in a narrow panel. Reading stderr at the moment of failure gets nothing: Bitween notices when the gRPC stream ends, while the reason is still being pumped on another thread. So SettledOutputAsync waits — bounded, ~2s — for the process to exit and the last error line to land. An adapter that is still alive is not waited on at all. The supervisor falls back to the same reader, and a start that throws outright is now recorded on the row instead of only reaching the node's log. Co-Authored-By: Claude Opus 5 --- SW.Bitween.Api/Resources/DataSources/Test.cs | 13 +- .../Services/Adapters/AdapterFailureReader.cs | 163 ++++++++++++++++++ .../DataSources/BusProviderSupervisor.cs | 46 ++++- .../AdapterFailureReaderTests.cs | 123 +++++++++++++ 4 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs create mode 100644 SW.Bitween.UnitTests/AdapterFailureReaderTests.cs diff --git a/SW.Bitween.Api/Resources/DataSources/Test.cs b/SW.Bitween.Api/Resources/DataSources/Test.cs index 76d80c3a..b577ed80 100644 --- a/SW.Bitween.Api/Resources/DataSources/Test.cs +++ b/SW.Bitween.Api/Resources/DataSources/Test.cs @@ -8,6 +8,7 @@ using SW.Bitween.Domain.Gateway; using SW.Bitween.Model; using SW.PrimitiveTypes; +using SW.Bitween.Services.Adapters; using SW.Serverless.Resident; namespace SW.Bitween.Resources.DataSources; @@ -74,9 +75,11 @@ public async Task Handle(int key, DataSourceTestRequest request) // and must not be disturbed by someone pressing Test. var instanceKey = $"test-{key}-{Guid.NewGuid():N}"[..24]; + ResidentAdapterInstance instance = null; + try { - var instance = await _adapters.StartExclusiveAsync(new AdapterSpec + instance = await _adapters.StartExclusiveAsync(new AdapterSpec { AdapterId = dataSource.AdapterId, InstanceKey = instanceKey, @@ -90,7 +93,13 @@ public async Task Handle(int key, DataSourceTestRequest request) { // A broker that cannot be reached throws on start, before any stage runs. That is a // result, not a server error: the operator asked whether it works, and it does not. - return Failed(ex.Message); + // + // What Bitween observes in that case is only "Adapter stream closed" — true, and no + // use at all. The adapter printed the actual reason before it died, so that is what + // this answers with; without it, the one control meant to save reading a log is the + // control that sends you to read one. + return Failed(AdapterFailureReader.Explain(ex.Message, + await AdapterFailureReader.SettledOutputAsync(instance))); } finally { diff --git a/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs b/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs new file mode 100644 index 00000000..0e052d0c --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using SW.Serverless.Resident; +using SW.Serverless.Sdk; + +namespace SW.Bitween.Services.Adapters; + +/// +/// Turns what a dying adapter printed into a sentence an operator can act on. +/// +/// An adapter that fails during startup — a broker refusing the credentials, TLS attempted against +/// a plaintext port — writes the reason to its own stderr and exits. The host keeps that output, +/// but nothing carried it anywhere a person looks: the connection test said "Adapter stream +/// closed", the data source row's LastException stayed null because that field only ever held what +/// a LIVE adapter reported about itself, and the actual exception existed solely in the API log. +/// +/// So the two questions an operator asks — "why did the test fail" and "why is this connection +/// down" — could only be answered by tailing a log on the node. This is what makes them +/// answerable from the screen that asked. +/// +public static class AdapterFailureReader +{ + /// Long enough for a real exception message, short enough to sit in a UI panel. + private const int MaxLength = 2000; + + /// + /// The most useful thing in the adapter's last output, or null when it said nothing. + /// + /// Prefers the last error it logged, because an adapter that dies logs the reason last. Falls + /// back to the final line of anything at all — an adapter can also die without using the SDK's + /// logger, and half an answer beats none. + /// + public static string Summarise(IEnumerable diagnostics) + { + var lines = (diagnostics ?? []).Where(l => !string.IsNullOrWhiteSpace(l)).ToList(); + if (lines.Count == 0) return null; + + var error = lines.LastOrDefault(l => l.Contains(Constants.LogErrorIdentifier, StringComparison.Ordinal)); + + return Clean(error ?? lines[^1]); + } + + /// + /// One error message with the adapter's last output appended, for a failure the caller already + /// has an exception for. The exception says what Bitween observed ("Adapter stream closed"); + /// the output says why, which is the half worth reading. + /// + public static string Explain(string message, IEnumerable diagnostics) + { + var detail = Summarise(diagnostics); + + if (string.IsNullOrWhiteSpace(detail)) return message; + if (string.IsNullOrWhiteSpace(message)) return detail; + + // Not the other way round: the adapter's reason is what an operator acts on, so it must + // survive being truncated in a narrow panel. + return $"{detail} ({message})"; + } + + /// + /// The adapter's output once it has finished arriving. + /// + /// Reading it the instant an invocation fails gets nothing: the failure is noticed when the + /// gRPC stream ends, while the reason travelled by stderr and is still being pumped on another + /// thread. Waiting for the process to exit is what makes the difference between "Adapter + /// stream closed" and the exception that closed it. + /// + /// The wait is short and bounded, and an adapter that is still alive — a timeout rather than a + /// crash — is not waited on at all. + /// + public static async Task> SettledOutputAsync(ResidentAdapterInstance instance, + int graceMilliseconds = 2000) + { + if (instance == null) return null; + + try + { + var process = instance.Process; + if (process is { HasExited: false }) + { + using var timeout = new CancellationTokenSource(graceMilliseconds); + try { await process.WaitForExitAsync(timeout.Token); } + catch (OperationCanceledException) { return instance.Diagnostics; } + } + + // Exited, but the last stderr lines may not have been pumped yet. They arrive in + // milliseconds; this waits for an error line rather than for the clock. + for (var attempt = 0; attempt < 20; attempt++) + { + if (instance.Diagnostics.Any(HasError)) break; + await Task.Delay(50); + } + + return instance.Diagnostics; + } + catch + { + // Never let reading the reason become the reason. + return instance.Diagnostics; + } + } + + private static bool HasError(string line) => + line != null && line.Contains(Constants.LogErrorIdentifier, StringComparison.Ordinal); + + /// + /// Undoes the SDK's line encoding. It escapes newlines so one log entry stays one line on the + /// wire, which leaves a stack trace as a single unreadable string unless it is put back. + /// + private static string Clean(string line) + { + if (string.IsNullOrWhiteSpace(line)) return null; + + var text = line + .Replace(Constants.LogErrorIdentifier, "", StringComparison.Ordinal) + .Replace(Constants.LogWarningIdentifier, "", StringComparison.Ordinal) + .Replace(Constants.LogInformationIdentifier, "", StringComparison.Ordinal) + .Replace(Constants.NewLineIdentifier, "\n", StringComparison.Ordinal) + .Replace(Constants.Delimiter, "\n", StringComparison.Ordinal) + .Trim(); + + // The SDK writes "message#!#exception#!#", so what is left is the message, then the + // exception's own first line — which is the type and text, and is the sentence to lead + // with. Everything past it is a stack trace nobody reads on a data source screen. + var parts = text.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0) return null; + + // The exception the adapter surfaced, then the innermost "--->" beneath it. That inner + // line is where the diagnosis lives: "None of the specified endpoints were reachable" is + // true of a wrong host, a wrong port and a firewall alike, while "Cannot determine the + // frame size" says TLS was attempted against a plaintext port and nothing else. + var lead = string.Join(" ", parts.Take(2).Select(StripTimestamp)); + + var cause = parts + .Where(l => l.StartsWith("--->", StringComparison.Ordinal)) + .Select(l => l[4..].Trim()) + .LastOrDefault(l => !string.IsNullOrWhiteSpace(l) && !lead.Contains(l, StringComparison.Ordinal)); + + var summary = cause == null ? lead : $"{lead} — caused by {cause}"; + + if (string.IsNullOrWhiteSpace(summary)) return null; + + return summary.Length > MaxLength ? summary[..MaxLength] + "…" : summary; + } + + /// + /// Drops the "12:34:56.789 " the host stamps on each captured line. Useful in a log tail, + /// noise at the front of a sentence on a form. + /// + private static string StripTimestamp(string line) + { + var space = line.IndexOf(' '); + if (space != 8 && space != 12) return line; + + var head = line[..space]; + return head.Count(c => c == ':') == 2 && head.All(c => char.IsDigit(c) || c == ':' || c == '.') + ? line[(space + 1)..] + : line; + } +} diff --git a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs index daacad84..9b335fea 100644 --- a/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs +++ b/SW.Bitween.Api/Services/DataSources/BusProviderSupervisor.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using SW.Bitween.Domain.DataSources; using SW.Bitween.Domain.Gateway; +using SW.Bitween.Services.Adapters; using SW.Bitween.Services.Cluster; using SW.Serverless.Resident; using System; @@ -166,6 +167,11 @@ await _adapters.StartExclusiveAsync(new AdapterSpec // One unreachable broker must not stop the others from starting. _logger.LogError(ex, "Could not start adapter {AdapterId} for data source {Name}.", dataSource.AdapterId, dataSource.Name); + + // And the operator who configured it learns why here rather than from the node's + // log. A start that throws leaves no instance to describe, so nothing else in this + // class would ever record it. + await RecordStartFailureAsync(dbContext, dataSource, ex, cancellationToken); } } @@ -286,6 +292,37 @@ private static string Fingerprint(DataSource dataSource, Dictionary kv.Key).Select(kv => $"{kv.Key}={kv.Value}")); + /// + /// Writes why a start failed onto the data source itself. + /// + /// A throw from StartExclusiveAsync leaves nothing running to describe, so the health + /// write-back below never sees it — which is how a data source ends up showing no connection + /// and no reason at the same time. The message is the one the operator would otherwise have + /// had to find in this node's log. + /// + private async Task RecordStartFailureAsync(BitweenDbContext dbContext, DataSource dataSource, + Exception exception, CancellationToken cancellationToken) + { + try + { + var row = await dbContext.Set() + .FirstOrDefaultAsync(d => d.Id == dataSource.Id, cancellationToken); + if (row == null) return; + + row.LastKnownState = "Failed"; + row.LastException = AdapterFailureReader.Explain(exception.Message, + _adapters.Get(dataSource.AdapterId, dataSource.Id.ToString())?.Diagnostics); + + await dbContext.SaveChangesAsync(cancellationToken); + } + catch (Exception ex) + { + // Recording why something failed must not become a second thing that fails. + _logger.LogWarning(ex, "Could not record the start failure for data source {Name}.", + dataSource.Name); + } + } + /// /// Health from the heartbeat, written back so the UI and the notifiers can see a broker that /// has gone away without anyone tailing logs. @@ -308,7 +345,14 @@ private async Task WriteBackHealthAsync(BitweenDbContext dbContext, Cancellation row.LastKnownState = instance.ReportedState ?? instance.State.ToString(); row.LastHeartbeatOn = instance.LastHeartbeatOn?.UtcDateTime; - row.LastException = instance.LastError; + + // LastError is what a LIVE adapter says about itself, so it is null for exactly the + // failure an operator most needs explained: one that died on startup and never got as + // far as reporting anything. The process printed why before it exited, so fall back to + // that — otherwise the screen shows "Quarantined" with no reason anywhere near it. + row.LastException = instance.LastError + ?? AdapterFailureReader.Summarise( + _adapters.Get(row.AdapterId, row.Id.ToString())?.Diagnostics); row.ConsecutiveFailures = instance.RestartCount; row.OwnedByNode = _leases.TryGetValue(row.Id, out var lease) ? $"{(_election as RabbitMqLeaderElection)?.NodeName ?? Environment.MachineName} (term {lease.Term})" diff --git a/SW.Bitween.UnitTests/AdapterFailureReaderTests.cs b/SW.Bitween.UnitTests/AdapterFailureReaderTests.cs new file mode 100644 index 00000000..2a04b3fd --- /dev/null +++ b/SW.Bitween.UnitTests/AdapterFailureReaderTests.cs @@ -0,0 +1,123 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Adapters; + +namespace SW.Bitween.UnitTests; + +/// +/// Reading an adapter's dying words. +/// +/// These are the exact lines a resident adapter writes when it cannot reach a broker. Before this +/// existed, all of it stayed in the node's log: the connection test answered "Adapter stream +/// closed" and the data source row's reason column stayed empty, so the one screen built to save +/// an operator from reading a log was the screen that sent them to read one. +/// +[TestClass] +public class AdapterFailureReaderTests +{ + /// The real shape: SDK marker, message, delimiter, then the exception with its newlines escaped. + private const string TlsOnPlainPort = + "{{log.error}}Resident adapter terminated.#!#RabbitMQ.Client.Exceptions.BrokerUnreachableException: " + + "None of the specified endpoints were reachable{{newline}} ---> " + + "System.Security.Authentication.AuthenticationException: Cannot determine the frame size or a " + + "corrupted frame was received.{{newline}} at System.Net.Security.SslStream.GetFrameSize#!#"; + + [TestMethod] + public void The_reason_survives_the_wire_encoding() + { + var summary = AdapterFailureReader.Summarise([TlsOnPlainPort]); + + // What the operator needs is the exception's own first line. The SDK escapes newlines so + // one log entry stays one line on the wire, which leaves this unreadable unless undone. + StringAssert.Contains(summary, "BrokerUnreachableException"); + StringAssert.Contains(summary, "None of the specified endpoints were reachable"); + Assert.IsFalse(summary.Contains("{{newline}}")); + Assert.IsFalse(summary.Contains("{{log.error}}")); + Assert.IsFalse(summary.Contains("#!#")); + } + + /// + /// The line that actually diagnoses it. "None of the specified endpoints were reachable" is + /// equally true of a wrong host, a wrong port and a firewall; "Cannot determine the frame + /// size" says TLS was attempted against a plaintext port, and nothing else does. + /// + [TestMethod] + public void The_innermost_cause_is_carried_too() + { + var summary = AdapterFailureReader.Summarise([TlsOnPlainPort]); + + StringAssert.Contains(summary, "caused by"); + StringAssert.Contains(summary, "Cannot determine the frame size"); + } + + /// The host stamps a time on each captured line; it is noise at the front of a sentence. + [TestMethod] + public void The_capture_timestamp_is_dropped() + { + var summary = AdapterFailureReader.Summarise(["10:27:28.223 " + TlsOnPlainPort]); + + StringAssert.StartsWith(summary, "Resident adapter terminated."); + } + + [TestMethod] + public void The_stack_trace_is_left_out() + { + var summary = AdapterFailureReader.Summarise([TlsOnPlainPort]); + + // A data source screen has room for the sentence, not the frames. + Assert.IsFalse(summary.Contains("at System.Net.Security")); + } + + [TestMethod] + public void The_last_error_wins_over_ordinary_chatter() + { + string[] output = + [ + "{{log.information}}Starting up", + "{{log.error}}First problem#!#System.Exception: one#!#", + "{{log.information}}Retrying", + "{{log.error}}Resident adapter terminated.#!#System.Exception: the one that killed it#!#" + ]; + + StringAssert.Contains(AdapterFailureReader.Summarise(output), "the one that killed it"); + } + + [TestMethod] + public void An_adapter_that_never_used_the_logger_still_says_something() + { + // Dying outside the SDK's logger is allowed — an unhandled exception on a background + // thread prints a bare stack trace — and half an answer beats none. + var summary = AdapterFailureReader.Summarise(["Unhandled exception. System.IO.IOException: pipe broken"]); + + StringAssert.Contains(summary, "IOException"); + } + + [TestMethod] + public void Silence_stays_silence() + { + Assert.IsNull(AdapterFailureReader.Summarise(null)); + Assert.IsNull(AdapterFailureReader.Summarise([])); + Assert.IsNull(AdapterFailureReader.Summarise([" ", ""])); + } + + /// + /// The reason leads and Bitween's own observation follows in brackets. That order is the + /// point: "Adapter stream closed" is what Bitween saw, and it is the half that explains + /// nothing, so it must be the half that gets truncated in a narrow panel. + /// + [TestMethod] + public void Explain_leads_with_the_adapters_reason() + { + var explained = AdapterFailureReader.Explain("Adapter stream closed.", [TlsOnPlainPort]); + + StringAssert.StartsWith(explained, "Resident adapter terminated."); + StringAssert.Contains(explained, "(Adapter stream closed.)"); + } + + [TestMethod] + public void Explain_falls_back_to_what_bitween_saw() + { + // No output at all — the process died before printing, or was killed outright. + Assert.AreEqual("Adapter stream closed.", + AdapterFailureReader.Explain("Adapter stream closed.", null)); + } +} From 7737e9a2a8ca09bfa15b76611fb5fc8ee6f0a2b6 Mon Sep 17 00:00:00 2001 From: Muhannad Al-Khatib Date: Mon, 7 Sep 2026 17:49:13 +0300 Subject: [PATCH 15/43] feat: a data source form is built from the adapter, not from the UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The front end carried its own copy of someone else's contract: which fields a RabbitMQ connection starts with, what DeclareMode accepts, which properties are credentials. A hand-kept copy drifts, and it did — the form advertised a "Tls" setting the adapter never read, so an operator could set it, see nothing wrong, and still be authenticating in the clear. QueueType, meanwhile, was a free-text box for a value the broker only accepts three of. An adapter now marks its options class [AdapterSettings(Kind, Label)] and each property [AdapterSetting(Hint/Default/AllowedValues/Secret/Required/Hidden)]. The contract is shared *source* rather than a package because adapters target net8.0 while the host is net10.0, and Bitween matches the attributes by name. DataSourceProviderCatalog reads them out of the published package with MetadataLoadContext — READ, never loaded, so nothing in a third party's adapter runs merely because Bitween described it — and GET /datasources/Providers serves the result. Add a setting to an adapter and it appears in the UI with no front-end change; delete the attributes and the form empties, which is the property the tests assert. Two things fall out of the same descriptor. Kind decides where a data source can be used: a bus gateway reads from a queue, so only a Broker can back one, enforced in Resources/BusGateways/{Create,Update} — the coming database adapters will be data sources too and must not be offerable there. And the detail page no longer wipes a half-typed setting on every poll: it reseeds the draft from a fingerprint of the editable fields rather than from object identity, which changed on every refetch whether or not anything had. Co-Authored-By: Claude Opus 5 --- .../RabbitOptions.cs | 41 +++ .../SW.Bitween.Adapters.Bus.RabbitMq.csproj | 5 + .../SW.Bitween.Adapters.Bus.Sqs.csproj | 5 + SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs | 28 ++ .../AdapterSettingAttribute.cs | 73 +++++ .../Resources/BusGateways/Create.cs | 18 +- .../Resources/BusGateways/Update.cs | 18 +- .../Resources/DataSources/Providers.cs | 39 +++ SW.Bitween.Api/SW.Bitween.Api.csproj | 4 + .../DataSources/DataSourceProviderCatalog.cs | 272 ++++++++++++++++++ .../Fixtures/BitweenFixture.cs | 1 + .../Tests/DataSourceApiTests.cs | 47 ++- .../Tests/DataSourceProviderCatalogTests.cs | 198 +++++++++++++ SW.Bitween.Sdk/Model/DataSource.cs | 54 ++++ SW.Bitween.Web/ClientApp/src/api/client.ts | 5 + .../ClientApp/src/api/http/dataSources.ts | 10 +- SW.Bitween.Web/ClientApp/src/api/queryKeys.ts | 2 + SW.Bitween.Web/ClientApp/src/api/types.ts | 30 ++ .../src/pages/bus-gateways/SourceDialog.tsx | 14 +- .../pages/data-sources/DataSourceNewPage.tsx | 53 +++- .../src/pages/data-sources/DataSourcePage.tsx | 98 ++++--- .../pages/data-sources/DataSourcesPage.tsx | 5 +- .../data-sources/__tests__/draft.test.ts | 73 +++++ .../data-sources/__tests__/providers.test.ts | 67 +++++ .../ClientApp/src/pages/data-sources/draft.ts | 38 +++ .../src/pages/data-sources/providers.ts | 126 ++++---- SW.Bitween.Web/Startup.cs | 5 + 27 files changed, 1196 insertions(+), 133 deletions(-) create mode 100644 SW.Bitween.Adapters.Shared/AdapterSettingAttribute.cs create mode 100644 SW.Bitween.Api/Resources/DataSources/Providers.cs create mode 100644 SW.Bitween.Api/Services/DataSources/DataSourceProviderCatalog.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/DataSourceProviderCatalogTests.cs create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/__tests__/draft.test.ts create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/__tests__/providers.test.ts create mode 100644 SW.Bitween.Web/ClientApp/src/pages/data-sources/draft.ts diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs index 8343403f..d9cd3d85 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs @@ -4,17 +4,41 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; /// Every one of these arrives as a DataSource property, bound by name. Nothing here is opinionated /// about how the broker should be laid out: the queue may already exist and be owned by someone /// else, or Bitween may declare it — decides which. +/// +/// The attributes are not documentation: Bitween reads them out of this assembly to build the form +/// an operator fills in, so a field added here appears there without anyone editing the UI. /// +[AdapterSettings( + Kind = "Broker", + Label = "RabbitMQ", + Description = "An AMQP broker the customer runs, separate from Bitween's own bus.")] public class RabbitOptions { + [AdapterSetting(Required = true, + Hint = "Host name only — no amqp:// or amqps:// prefix, and no credentials.")] public string Host { get; set; } = "localhost"; + + [AdapterSetting(Default = "5672", + Hint = "5672 plain, 5671 for TLS. A hosted broker is almost always 5671.")] public int Port { get; set; } = 5672; + + [AdapterSetting(Required = true)] public string UserName { get; set; } = "guest"; + + [AdapterSetting(Secret = true, Required = true)] public string Password { get; set; } + + [AdapterSetting(Default = "/", + Hint = "On a shared hosted broker this is usually the same as the user name.")] public string VirtualHost { get; set; } = "/"; + + [AdapterSetting(Default = "false", AllowedValues = new[] {"true", "false"}, + Hint = "Set for anything that is not localhost — AMQP authenticates in the clear otherwise. " + + "Pair it with port 5671.")] public bool UseSsl { get; set; } /// Comma-separated queue names, supplied by the gateways bound to this data source. + [AdapterSetting(Hidden = true)] public string Endpoints { get; set; } /// @@ -22,24 +46,41 @@ public class RabbitOptions /// assert — verify it exists and fail loudly if not (passive declare). /// create — declare queues, and an exchange and binding when Exchange is set. /// + [AdapterSetting(Default = "assert", AllowedValues = new[] {"none", "assert", "create"}, + Hint = "none — assume everything exists. assert — check and fail loudly if not. " + + "create — declare the queues.")] public string DeclareMode { get; set; } = "assert"; /// Optional. When set, each endpoint queue is bound to it. + [AdapterSetting(Hint = "Optional. When set, each endpoint queue is bound to it.")] public string Exchange { get; set; } + + [AdapterSetting(Default = "topic", AllowedValues = new[] {"direct", "fanout", "topic", "headers"}, + Hint = "Only used when Exchange is set.")] public string ExchangeType { get; set; } = "topic"; + + [AdapterSetting(Hint = "Binding key for the exchange. Only used when Exchange is set.")] public string RoutingKey { get; set; } /// /// False connects and declares but does not consume. This is what a connection test runs as: /// without it, testing a data source would start pulling messages off the customer's queue. /// + [AdapterSetting(Hidden = true)] public bool Consume { get; set; } = true; /// Broker-side backpressure; pairs with the host's credit window. + [AdapterSetting(Default = "16", + Hint = "How many messages the broker lets Bitween hold unacknowledged at once.")] public ushort Prefetch { get; set; } = 16; + [AdapterSetting(Default = "true", AllowedValues = new[] {"true", "false"}, + Hint = "Declared queues survive a broker restart. Only read when DeclareMode is create.")] public bool Durable { get; set; } = true; /// Passed straight through as x-queue-type: classic, quorum or stream. + [AdapterSetting(AllowedValues = new[] {"classic", "quorum", "stream"}, + Hint = "Passed through as x-queue-type when Bitween declares the queue. Leave empty to let " + + "the broker choose. It cannot be changed on a queue that already exists.")] public string QueueType { get; set; } } diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj index e0f27836..32528de3 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj +++ b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj @@ -11,4 +11,9 @@ + + + + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj index e906aa8d..cdd55524 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj +++ b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj @@ -9,4 +9,9 @@ + + + + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs index b88730f0..ccfa419f 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs @@ -1,43 +1,71 @@ namespace SW.Bitween.Adapters.Bus.Sqs; +/// +/// Bound by name from the DataSource's properties. The attributes are what Bitween's data source +/// form is built from — see AdapterSettingAttribute for why the UI no longer keeps its own copy of +/// this list. +/// +[AdapterSettings( + Kind = "Broker", + Label = "Amazon SQS", + Description = "An SQS queue, including the one an Amazon Selling Partner notification " + + "subscription delivers to.")] public class SqsOptions { /// /// False creates the client but starts no pollers. This is what a connection test runs as: /// without it, testing a data source would start receiving from the customer's queue. /// + [AdapterSetting(Hidden = true)] public bool Consume { get; set; } = true; + [AdapterSetting(Required = true, Default = "eu-west-1", + Hint = "The queue's region, not Bitween's.")] public string Region { get; set; } = "eu-west-1"; /// /// Leave both blank to use the ambient credential chain — instance profile, IRSA, or the /// environment. That is the right answer on AWS; explicit keys are for everything else. /// + [AdapterSetting(Secret = true, + Hint = "Leave both keys blank on AWS to use the instance profile or IRSA instead.")] public string AccessKeyId { get; set; } + + [AdapterSetting(Secret = true)] public string SecretAccessKey { get; set; } /// Override for LocalStack or ElasticMQ in development. + [AdapterSetting(Hint = "Override for LocalStack or ElasticMQ in development. Leave empty for AWS.")] public string ServiceUrl { get; set; } /// Comma-separated queue URLs, supplied by the gateways bound to this data source. + [AdapterSetting(Hidden = true)] public string Endpoints { get; set; } /// Long polling. 20 is the maximum and the only sensible value — 0 burns money and API calls. + [AdapterSetting(Default = "20", + Hint = "Long polling, in seconds. 20 is the maximum and the only sensible value — 0 burns " + + "money and API calls.")] public int WaitTimeSeconds { get; set; } = 20; /// Max 10 per receive; that is an SQS limit, not a choice. + [AdapterSetting(Default = "10", Hint = "10 is SQS's own maximum, not a choice.")] public int MaxMessagesPerReceive { get; set; } = 10; /// /// How long a received message stays invisible to other consumers. It must exceed the time /// Bitween takes to persist, or the message is redelivered while still being handled. /// + [AdapterSetting(Default = "60", + Hint = "Must exceed how long Bitween takes to persist a message, or SQS redelivers one " + + "already being handled.")] public int VisibilityTimeoutSeconds { get; set; } = 60; /// /// SP-API wraps its notifications in an envelope. On, the adapter forwards only the payload /// and promotes notificationType and the SP-API metadata into headers. /// + [AdapterSetting(Default = "false", AllowedValues = new[] { "true", "false" }, + Hint = "true unwraps the SP-API envelope so subscriptions see the notification payload itself.")] public bool UnwrapSellingPartnerNotification { get; set; } } diff --git a/SW.Bitween.Adapters.Shared/AdapterSettingAttribute.cs b/SW.Bitween.Adapters.Shared/AdapterSettingAttribute.cs new file mode 100644 index 00000000..01caee6b --- /dev/null +++ b/SW.Bitween.Adapters.Shared/AdapterSettingAttribute.cs @@ -0,0 +1,73 @@ +using System; + +namespace SW.Bitween.Adapters; + +/// +/// Marks the type that describes an adapter's connection settings — one per adapter. +/// +/// The point is that the UI stops knowing anything about a particular broker. Before this, the +/// fields a data source form offered, their defaults and their allowed values were a hand-written +/// table in the front end, which meant the form and the adapter could disagree silently: a hint +/// told operators to set "Tls" for a very long time while the adapter only ever read "UseSsl", so +/// a connection that looked encrypted in the UI was in the clear on the wire. +/// +/// The adapter is the only thing that knows what it accepts, so it is the thing that says so. +/// +[AttributeUsage(AttributeTargets.Class)] +public sealed class AdapterSettingsAttribute : Attribute +{ + /// What to call this provider in a menu. Falls back to the adapter id. + public string Label { get; set; } + + /// + /// What kind of thing this connects to, from Bitween's DataSourceKind vocabulary: Broker, + /// Relational, Document, ObjectStore or Http. It is not decoration — a bus gateway can only + /// read from a Broker, so a resident adapter that holds a database connection must not be + /// offerable as one. Unstated means Broker, which is all that existed when this was added. + /// + public string Kind { get; set; } = "Broker"; + + /// One line, shown under the provider chooser. + public string Description { get; set; } +} + +/// +/// One operator-visible setting, declared beside the property that consumes it. +/// +/// Every public read/write property of an type is a setting +/// whether or not it carries this attribute — nothing an adapter binds is invisible to the person +/// configuring it. The attribute adds what reflection cannot see: why the field exists, what values +/// are legal, whether it holds a credential, and what the default is (a property initialiser lives +/// in IL that is deliberately never executed here). +/// +[AttributeUsage(AttributeTargets.Property)] +public sealed class AdapterSettingAttribute : Attribute +{ + /// Shown under the input. Say what it is for, not what its type is. + public string Hint { get; set; } + + /// + /// The value a new data source starts with. Stated rather than inferred: the describer reads + /// the assembly without running it, so a property initialiser is not observable. + /// + public string Default { get; set; } + + /// + /// The complete set of legal values. Present means the UI offers a choice instead of a text + /// box, which is the difference between "quorum" and a typo the broker rejects at declare time. + /// + public string[] AllowedValues { get; set; } + + /// Holds a credential: masked in responses and never shown back. + public bool Secret { get; set; } + + /// Offered on the create form and flagged when left empty. + public bool Required { get; set; } + + /// + /// Supplied by Bitween itself, not by an operator — endpoints come from the gateways bound to + /// the data source, and Consume is how a connection test avoids draining a live queue. + /// Hidden settings are not offered as fields at all. + /// + public bool Hidden { get; set; } +} diff --git a/SW.Bitween.Api/Resources/BusGateways/Create.cs b/SW.Bitween.Api/Resources/BusGateways/Create.cs index 260211c0..595e0cfe 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Create.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Create.cs @@ -56,11 +56,23 @@ internal static async Task EnsureDataSourceAsync(BitweenDbContext dbContext, Bus { if (model.DataSourceId == null) return; - var exists = await dbContext.Set() - .AnyAsync(d => d.Id == model.DataSourceId); - if (!exists) + // FirstOrDefaultAsync rather than a projection: FluentValidation is in scope here and + // its own Where extension wins the overload. + var dataSource = await dbContext.Set() + .AsNoTracking() + .FirstOrDefaultAsync(d => d.Id == model.DataSourceId); + if (dataSource == null) throw new SWNotFoundException($"DataSource with Id {model.DataSourceId} not found"); + // A data source is not only a broker connection — a resident adapter holding a database + // session is one too — and only a broker has queues to consume. Reading this from the + // kind rather than from the adapter id keeps the rule true for providers nobody has + // written yet. + if (dataSource.Kind != Domain.DataSources.DataSourceKind.Broker) + throw new SWException( + $"Data source {model.DataSourceId} is a {dataSource.Kind} connection. A bus " + + "gateway reads messages from a queue or topic, so it can only use a Broker."); + if (string.IsNullOrWhiteSpace(model.Endpoint)) throw new SWException( "An external bus gateway needs an endpoint — the queue, topic or subscription " diff --git a/SW.Bitween.Api/Resources/BusGateways/Update.cs b/SW.Bitween.Api/Resources/BusGateways/Update.cs index ad0a8294..1be51830 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Update.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Update.cs @@ -51,11 +51,23 @@ private static async Task EnsureEndpointFreeAsync( { if (model.DataSourceId == null) return; - var exists = await dbContext.Set() - .AnyAsync(d => d.Id == model.DataSourceId); - if (!exists) + // FirstOrDefaultAsync rather than a projection: FluentValidation is in scope here and + // its own Where extension wins the overload. + var dataSource = await dbContext.Set() + .AsNoTracking() + .FirstOrDefaultAsync(d => d.Id == model.DataSourceId); + if (dataSource == null) throw new SWNotFoundException($"DataSource with Id {model.DataSourceId} not found"); + // A data source is not only a broker connection — a resident adapter holding a database + // session is one too — and only a broker has queues to consume. Reading this from the + // kind rather than from the adapter id keeps the rule true for providers nobody has + // written yet. + if (dataSource.Kind != Domain.DataSources.DataSourceKind.Broker) + throw new SWException( + $"Data source {model.DataSourceId} is a {dataSource.Kind} connection. A bus " + + "gateway reads messages from a queue or topic, so it can only use a Broker."); + if (string.IsNullOrWhiteSpace(model.Endpoint)) throw new SWException( "An external bus gateway needs an endpoint — the queue, topic or subscription " diff --git a/SW.Bitween.Api/Resources/DataSources/Providers.cs b/SW.Bitween.Api/Resources/DataSources/Providers.cs new file mode 100644 index 00000000..641acb36 --- /dev/null +++ b/SW.Bitween.Api/Resources/DataSources/Providers.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using SW.Bitween.Model; +using SW.Bitween.Services.DataSources; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.DataSources; + +/// +/// The data source providers this deployment can offer, and what each one accepts. +/// +/// Serving this from the adapters rather than from the front end is the whole point: a provider +/// added, a setting renamed, or a new allowed value appears in the UI the moment the adapter is +/// published, and cannot silently disagree with what the adapter actually reads. +/// +[HandlerName("Providers")] +public class Providers : IQueryHandler +{ + private readonly BitweenDbContext _dbContext; + private readonly RequestContext _requestContext; + private readonly DataSourceProviderCatalog _catalog; + + public Providers(BitweenDbContext dbContext, RequestContext requestContext, + DataSourceProviderCatalog catalog) + { + _dbContext = dbContext; + _requestContext = requestContext; + _catalog = catalog; + } + + public async Task Handle() + { + // Viewing is enough: this describes what Bitween can connect to, not what it is connected + // to. Nothing here comes from a data source, so there is no credential to leak. + await _requestContext.EnsurePermission(_dbContext, Model.Permissions.DataSources.View); + + return await _catalog.ListAsync(); + } +} diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj index eb729916..29fd6892 100644 --- a/SW.Bitween.Api/SW.Bitween.Api.csproj +++ b/SW.Bitween.Api/SW.Bitween.Api.csproj @@ -48,4 +48,8 @@ + + + + diff --git a/SW.Bitween.Api/Services/DataSources/DataSourceProviderCatalog.cs b/SW.Bitween.Api/Services/DataSources/DataSourceProviderCatalog.cs new file mode 100644 index 00000000..2bc527d8 --- /dev/null +++ b/SW.Bitween.Api/Services/DataSources/DataSourceProviderCatalog.cs @@ -0,0 +1,272 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using SW.Serverless; + +namespace SW.Bitween.Services.DataSources; + +/// +/// What each data source provider accepts, read out of the adapter itself. +/// +/// The front end used to carry this list by hand — which fields a RabbitMQ data source starts with, +/// what DeclareMode allows, which values are credentials. A hand-written copy of someone else's +/// contract drifts: the form advertised a "Tls" field for months that the adapter never read, so an +/// operator could tick it, see nothing wrong, and still be authenticating in the clear. +/// +/// So the adapter declares its own settings (AdapterSettingsAttribute) and this reads them. +/// Two properties of how it reads matter: +/// +/// - The assembly is inspected with , never loaded. Nothing in the +/// adapter runs. That is what makes describing safe for a resident bus provider: asking a running +/// one would mean an extra connection to a customer's broker, and asking a stopped one would mean +/// starting a connection nobody asked for. +/// - The attributes are matched by NAME, not by type identity. Adapters build against their own +/// copy of the contract — they target net8.0 while the host is on net10.0 — and a type loaded in +/// a metadata context is never reference-equal to the one the host compiled against anyway. +/// +public class DataSourceProviderCatalog +{ + /// + /// Where Bitween's own providers live. A third-party adapter is found by the Kind stamped on + /// it at publish time instead; this prefix is what finds the ones published before stamping + /// existed, and it is a prefilter either way — describing every adapter in the store to build + /// a menu would mean downloading every adapter in the store. + /// + public const string ConventionPrefix = "bitween."; + + private const string SettingsAttribute = "AdapterSettingsAttribute"; + private const string SettingAttribute = "AdapterSettingAttribute"; + /// What the installer stamps on an adapter that connects to something. + private static readonly string[] ProviderKinds = ["bus", "datasource"]; + + private readonly AdapterInstaller _installer; + private readonly ICloudFilesService _cloudFiles; + private readonly ServerlessOptions _options; + private readonly IMemoryCache _cache; + private readonly ILogger _logger; + + public DataSourceProviderCatalog(AdapterInstaller installer, ICloudFilesService cloudFiles, + ServerlessOptions options, IMemoryCache cache, ILogger logger) + { + _installer = installer; + _cloudFiles = cloudFiles; + _options = options; + _cache = cache; + _logger = logger; + } + + /// + /// Every data source provider this deployment can offer, described, optionally narrowed to one + /// DataSourceKind. An adapter that cannot be read is left out rather than throwing: one broken + /// package must not empty the provider menu. + /// + public async Task> ListAsync(string kind = null) + { + var described = new List(); + + foreach (var adapterId in await CandidatesAsync()) + { + var descriptor = await DescribeAsync(adapterId); + if (descriptor == null) continue; + + if (kind != null && !string.Equals(descriptor.Kind, kind, StringComparison.OrdinalIgnoreCase)) + continue; + + described.Add(descriptor); + } + + return described.OrderBy(p => p.Label, StringComparer.OrdinalIgnoreCase).ToList(); + } + + /// + /// One provider, or null when the adapter is missing, unreadable, or declares no settings. + /// Cached against the package hash, so a republished adapter re-describes itself and an + /// unchanged one is read from disk once. + /// + public async Task DescribeAsync(string adapterId) + { + try + { + var installed = await _installer.GetMetadataAsync(adapterId); + var cacheKey = $"bus-provider.{adapterId}.{installed.Hash}"; + if (_cache.TryGetValue(cacheKey, out DataSourceProviderDescriptor cached)) return cached; + + // LocalPath is the entry assembly's full path; it exists only once extracted. + await _installer.InstallAsync(adapterId); + + var descriptor = Describe(adapterId, installed.LocalPath); + if (descriptor == null) return null; + + return _cache.Set(cacheKey, descriptor, TimeSpan.FromMinutes(30)); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not describe bus provider {AdapterId}; leaving it out of " + + "the catalog.", adapterId); + return null; + } + } + + private async Task> CandidatesAsync() + { + var root = _options.AdapterRemotePath; + var candidates = new SortedSet(StringComparer.OrdinalIgnoreCase); + + foreach (var item in await _cloudFiles.ListAsync($"{root}/")) + { + if (item.Size <= 0) continue; + + var adapterId = item.Key.StartsWith($"{root}/", StringComparison.OrdinalIgnoreCase) + ? item.Key[(root.Length + 1)..] + : item.Key; + + // Versioned uploads keep the adapter id one segment up from the version. + if (Semver.IsVersionNumber(adapterId.Split('/').Last())) + adapterId = string.Join('/', adapterId.Split('/')[..^1]); + + if (adapterId.StartsWith(ConventionPrefix, StringComparison.OrdinalIgnoreCase)) + { + candidates.Add(adapterId); + continue; + } + + if (await DeclaresProviderKindAsync(adapterId)) candidates.Add(adapterId); + } + + return candidates.ToList(); + } + + private async Task DeclaresProviderKindAsync(string adapterId) + { + try + { + var metadata = await _installer.GetMetadataAsync(adapterId); + return metadata?.AdapterValues != null && + metadata.AdapterValues.TryGetValue("Kind", out var kinds) && + kinds.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Any(k => ProviderKinds.Contains(k, StringComparer.OrdinalIgnoreCase)); + } + catch + { + return false; + } + } + + /// + /// Reads one extracted adapter. Public so a test can point it at a publish directory without + /// a cloud store in the way. + /// + public static DataSourceProviderDescriptor Describe(string adapterId, string entryAssemblyPath) + { + if (!File.Exists(entryAssemblyPath)) return null; + + var directory = Path.GetDirectoryName(entryAssemblyPath)!; + var assemblies = Directory.GetFiles(directory, "*.dll", SearchOption.AllDirectories) + .Concat(Directory.GetFiles(Path.GetDirectoryName(typeof(object).Assembly.Location)!, "*.dll")) + .Distinct() + .ToList(); + + using var context = new MetadataLoadContext(new PathAssemblyResolver(assemblies)); + var assembly = context.LoadFromAssemblyPath(entryAssemblyPath); + + foreach (var type in SafeTypes(assembly)) + { + var marker = AttributeNamed(type.GetCustomAttributesData(), SettingsAttribute); + if (marker == null) continue; + + return new DataSourceProviderDescriptor + { + AdapterId = adapterId, + Label = Named(marker, "Label") ?? adapterId, + Kind = Named(marker, "Kind") ?? "Broker", + Description = Named(marker, "Description"), + Settings = SettingsOf(type) + }; + } + + return null; + } + + private static List SettingsOf(Type type) + { + var settings = new List(); + + foreach (var property in type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (!property.CanRead || !property.CanWrite) continue; + + var attribute = AttributeNamed(property.GetCustomAttributesData(), SettingAttribute); + if (attribute != null && Named(attribute, "Hidden")) continue; + + settings.Add(new DataSourceProviderSetting + { + Name = property.Name, + Type = TypeOf(property.PropertyType), + Hint = attribute == null ? null : Named(attribute, "Hint"), + Default = attribute == null ? null : Named(attribute, "Default"), + AllowedValues = attribute == null ? null : NamedArray(attribute, "AllowedValues"), + Secret = attribute != null && Named(attribute, "Secret"), + Required = attribute != null && Named(attribute, "Required") + }); + } + + return settings; + } + + /// + /// What kind of input this is. Deliberately coarse: the UI needs to choose between a text box, + /// a number, a toggle and a menu, and anything finer would be the UI knowing about brokers + /// again. + /// + private static string TypeOf(Type type) + { + var name = type.FullName ?? type.Name; + + if (name == typeof(bool).FullName) return DataSourceProviderSetting.BooleanType; + + return name is not null && ( + name == typeof(int).FullName || name == typeof(long).FullName || + name == typeof(short).FullName || name == typeof(ushort).FullName || + name == typeof(uint).FullName || name == typeof(ulong).FullName || + name == typeof(byte).FullName || name == typeof(double).FullName || + name == typeof(decimal).FullName) + ? DataSourceProviderSetting.NumberType + : DataSourceProviderSetting.StringType; + } + + private static CustomAttributeData AttributeNamed(IEnumerable attributes, string name) => + attributes.FirstOrDefault(a => + string.Equals(a.AttributeType.Name, name, StringComparison.Ordinal)); + + private static T Named(CustomAttributeData attribute, string name) + { + var argument = attribute.NamedArguments + .FirstOrDefault(a => string.Equals(a.MemberName, name, StringComparison.Ordinal)); + + return argument.TypedValue.Value is T value ? value : default; + } + + private static string[] NamedArray(CustomAttributeData attribute, string name) + { + var argument = attribute.NamedArguments + .FirstOrDefault(a => string.Equals(a.MemberName, name, StringComparison.Ordinal)); + + return argument.TypedValue.Value is IEnumerable values + ? values.Select(v => v.Value as string).Where(v => v != null).ToArray() + : null; + } + + private static IEnumerable SafeTypes(Assembly assembly) + { + try { return assembly.GetTypes(); } + catch (ReflectionTypeLoadException ex) { return ex.Types.Where(t => t != null); } + catch { return []; } + } +} diff --git a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs index 8b0f8838..c51d7bde 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs @@ -247,6 +247,7 @@ await Task.WhenAll(_postgres.StartAsync(), _rabbitMq.StartAsync(), _mailHog.Star services.AddSingleton(); services.AddScoped(); + services.AddSingleton(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); diff --git a/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs b/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs index d37f61bd..762c5773 100644 --- a/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/DataSourceApiTests.cs @@ -498,12 +498,55 @@ public async Task Inspecting_a_connection_this_node_does_not_hold_says_so() Assert.NotNull(result.Error); } + /// + /// A data source is not only a broker: a resident adapter holding a database session is one + /// too, and one of those has no queue for a gateway to consume. Refusing it here matters more + /// than the menu that hides it — the menu is a courtesy, this is the rule. + /// + [Fact] + public async Task A_bus_gateway_cannot_read_from_a_non_broker_data_source() + { + var dataSourceId = await CreateAsync(new Dictionary(), kind: "Relational"); + var documentId = await CreateDocumentAsync(); + + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var create = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + var error = await Assert.ThrowsAnyAsync(() => create.Handle(new BusGatewayCreate + { + Name = Unique("gw"), + DocumentId = documentId, + DataSourceId = dataSourceId, + Endpoint = "orders" + })); + + Assert.Contains("Broker", error.Message, StringComparison.Ordinal); + } + + /// + /// And the same rule on the way in through an edit, which is the path that would otherwise + /// move a working gateway onto a database connection. + /// + [Fact] + public async Task A_bus_gateway_cannot_be_moved_onto_a_non_broker_data_source() + { + var broker = await CreateAsync(new Dictionary()); + var database = await CreateAsync(new Dictionary(), kind: "Relational"); + var gatewayId = await CreateGatewayAsync(broker, Unique("q")); + + var error = await Assert.ThrowsAnyAsync( + () => UpdateGatewayAsync(gatewayId, database, "orders")); + + Assert.Contains("Broker", error.Message, StringComparison.Ordinal); + } + // ---------------------------------------------------------------- helpers private static string Unique(string prefix) => $"{prefix}-{Guid.NewGuid():N}"[..20]; private async Task CreateAsync(Dictionary properties, - string name = null, List secretProperties = null) + string name = null, List secretProperties = null, string kind = "Broker") { await using var scope = _fixture.CreateScope(); scope.Superuser(); @@ -513,7 +556,7 @@ private async Task CreateAsync(Dictionary properties, { Name = name ?? Unique("ds"), AdapterId = BusAdapters.RabbitMq, - Kind = "Broker", + Kind = kind, Properties = properties, SecretProperties = secretProperties ?? ["Password"] }); diff --git a/SW.Bitween.IntegrationTests/Tests/DataSourceProviderCatalogTests.cs b/SW.Bitween.IntegrationTests/Tests/DataSourceProviderCatalogTests.cs new file mode 100644 index 00000000..466cfea8 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/DataSourceProviderCatalogTests.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using SW.PrimitiveTypes; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.Bitween.Services.DataSources; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// What a data source form is built from. +/// +/// The front end used to carry this itself: which fields a RabbitMQ connection starts with, what +/// DeclareMode accepts, which properties are credentials. A hand-kept copy of someone else's +/// contract drifts, and it did — the form advertised a "Tls" setting the adapter never read, so an +/// operator could set it, see nothing wrong, and still be authenticating in the clear. +/// +/// These tests are about that failure mode. They assert the description comes from the adapter +/// package rather than from anything Bitween wrote down, which is the only property that stops the +/// two disagreeing again. +/// +[Collection("Bitween")] +public class DataSourceProviderCatalogTests +{ + private readonly BitweenFixture _fixture; + + public DataSourceProviderCatalogTests(BitweenFixture fixture) => _fixture = fixture; + + /// + /// The adapters the fixture installed describe themselves. Nothing in Bitween names these + /// settings — delete the attributes from RabbitOptions and this list empties. + /// + [Fact] + public async Task A_provider_describes_its_own_settings() + { + var rabbit = await DescribeAsync(BusAdapters.RabbitMq); + + Assert.NotNull(rabbit); + Assert.Equal("RabbitMQ", rabbit.Label); + Assert.Equal("Broker", rabbit.Kind); + + var names = rabbit.Settings.Select(s => s.Name).ToList(); + Assert.Contains("Host", names); + Assert.Contains("UseSsl", names); + Assert.Contains("QueueType", names); + } + + /// + /// The point of the whole exercise: a value the UI could only have guessed at comes from the + /// adapter, so the menu it renders cannot offer a queue type RabbitMQ would reject. + /// + [Fact] + public async Task Allowed_values_come_from_the_adapter() + { + var rabbit = await DescribeAsync(BusAdapters.RabbitMq); + + var queueType = rabbit.Settings.Single(s => s.Name == "QueueType"); + Assert.Equal(["classic", "quorum", "stream"], queueType.AllowedValues); + + var declareMode = rabbit.Settings.Single(s => s.Name == "DeclareMode"); + Assert.Equal(["none", "assert", "create"], declareMode.AllowedValues); + Assert.Equal("assert", declareMode.Default); + } + + /// + /// The setting whose absence started this. It must be named UseSsl — what the adapter binds — + /// and not the "Tls" the old hand-written hint told operators to add. + /// + [Fact] + public async Task Tls_is_described_by_the_name_the_adapter_actually_binds() + { + var rabbit = await DescribeAsync(BusAdapters.RabbitMq); + + Assert.Contains(rabbit.Settings, s => s.Name == "UseSsl"); + Assert.DoesNotContain(rabbit.Settings, s => s.Name == "Tls"); + } + + /// + /// A credential is declared as one by the adapter, so the form masks it before anything has + /// been saved — without the name having to match a heuristic. + /// + [Fact] + public async Task A_credential_is_declared_secret_by_the_adapter() + { + var rabbit = await DescribeAsync(BusAdapters.RabbitMq); + var sqs = await DescribeAsync(BusAdapters.Sqs); + + Assert.True(rabbit.Settings.Single(s => s.Name == "Password").Secret); + Assert.True(sqs.Settings.Single(s => s.Name == "SecretAccessKey").Secret); + Assert.False(rabbit.Settings.Single(s => s.Name == "Host").Secret); + } + + /// + /// Endpoints come from the gateways bound to the data source and Consume is how a connection + /// test avoids draining a live queue. Neither is an operator's to set, so neither is offered. + /// + [Fact] + public async Task Host_supplied_settings_are_not_offered_as_fields() + { + var rabbit = await DescribeAsync(BusAdapters.RabbitMq); + + Assert.DoesNotContain(rabbit.Settings, s => s.Name == "Endpoints"); + Assert.DoesNotContain(rabbit.Settings, s => s.Name == "Consume"); + } + + /// + /// A number is described as one. Coarse on purpose — it decides which input to render, and + /// anything finer would be the UI knowing about brokers again. + /// + [Fact] + public async Task A_setting_carries_enough_type_to_pick_an_input() + { + var rabbit = await DescribeAsync(BusAdapters.RabbitMq); + + Assert.Equal(DataSourceProviderSetting.NumberType, + rabbit.Settings.Single(s => s.Name == "Prefetch").Type); + Assert.Equal(DataSourceProviderSetting.BooleanType, + rabbit.Settings.Single(s => s.Name == "UseSsl").Type); + Assert.Equal(DataSourceProviderSetting.StringType, + rabbit.Settings.Single(s => s.Name == "Host").Type); + } + + /// + /// Both installed providers turn up through the resource an operator's browser calls, which is + /// what makes the menu self-populating rather than a list someone maintains. + /// + [Fact] + public async Task The_resource_lists_every_installed_provider() + { + await using var scope = _fixture.CreateScope(); + scope.Superuser(); + var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + var providers = (List)await handler.Handle(); + var ids = providers.Select(p => p.AdapterId).ToList(); + + Assert.Contains(BusAdapters.RabbitMq, ids); + Assert.Contains(BusAdapters.Sqs, ids); + Assert.All(providers, p => Assert.NotEmpty(p.Settings)); + } + + /// + /// An adapter that is not a provider has nothing to describe, and asking about one that was + /// never installed is not an error either — a broken or absent package must not empty the menu + /// for the ones that are fine. + /// + [Fact] + public async Task An_adapter_that_declares_nothing_is_simply_not_a_provider() + { + var catalog = _fixture.App.Services.GetRequiredService(); + + Assert.Null(await catalog.DescribeAsync("no.such.adapter.at.all")); + } + + /// + /// A provider nobody at Simplify9 published. Bitween's own adapters are found by their + /// "bitween." id prefix, which cannot possibly find a third party's — so the installer stamps + /// what an adapter IS onto the package (SW-Serverless #117, from its [AdapterKind]), and the + /// catalog reads that instead of the name. + /// + /// The id here deliberately shares nothing with the convention: if this adapter is offered, + /// only the stamp can explain it. + /// + [Fact] + public async Task A_third_party_provider_is_found_by_its_stamped_kind_not_its_name() + { + const string adapterId = "acme.connectors.widgetbus"; + + using var scope = _fixture.App.Services.CreateScope(); + await AdapterInstaller.InstallAsync( + scope.ServiceProvider.GetRequiredService(), + "SW.Bitween.Adapters.Bus.RabbitMq", adapterId, + "SW.Bitween.Adapters.Bus.RabbitMq.dll", + new Dictionary + { + ["Protocol"] = "2", ["Lifecycle"] = "resident", ["Kind"] = "bus" + }); + + var catalog = _fixture.App.Services.GetRequiredService(); + var offered = await catalog.ListAsync(); + + Assert.Contains(offered, p => p.AdapterId == adapterId); + Assert.False(adapterId.StartsWith(DataSourceProviderCatalog.ConventionPrefix)); + } + + private async Task DescribeAsync(string adapterId) + { + var catalog = _fixture.App.Services.GetRequiredService(); + var descriptor = await catalog.DescribeAsync(adapterId); + + Assert.NotNull(descriptor); + return descriptor; + } +} diff --git a/SW.Bitween.Sdk/Model/DataSource.cs b/SW.Bitween.Sdk/Model/DataSource.cs index 8edf2b8c..ec0a6ba7 100644 --- a/SW.Bitween.Sdk/Model/DataSource.cs +++ b/SW.Bitween.Sdk/Model/DataSource.cs @@ -188,3 +188,57 @@ public class DataSourceInspectResult public string Error { get; set; } } + +/// +/// What a bus provider accepts, as the adapter itself declares it. +/// +/// This exists so the UI can render a data source form without knowing anything about brokers. The +/// alternative — the one this replaced — is a table of fields, defaults and allowed values kept by +/// hand in the front end, which is a copy of a contract it does not own and cannot be told when it +/// changes. +/// +public class DataSourceProviderDescriptor +{ + public string AdapterId { get; set; } + + /// What to call it in a menu. The adapter id when the adapter did not say. + public string Label { get; set; } + + /// + /// Which DataSourceKind this provider produces — Broker, Relational, Document, ObjectStore or + /// Http. A data source is not only a broker connection: a resident adapter that holds a + /// database session is one too, and it must never be offered as a bus gateway's source. + /// + public string Kind { get; set; } = "Broker"; + + public string Description { get; set; } + + public List Settings { get; set; } = []; +} + +/// One connection setting an operator can fill in. +public class DataSourceProviderSetting +{ + public const string StringType = "string"; + public const string NumberType = "number"; + public const string BooleanType = "boolean"; + + /// The property name the adapter binds by — this is the data source property key. + public string Name { get; set; } + + /// string, number or boolean. Coarse on purpose: it picks an input, nothing more. + public string Type { get; set; } = StringType; + + public string Hint { get; set; } + + /// What a new data source starts with. Null means start it empty. + public string Default { get; set; } + + /// When set, the only legal values — the UI offers these instead of free text. + public string[] AllowedValues { get; set; } + + /// Masked in responses and never shown back once stored. + public bool Secret { get; set; } + + public bool Required { get; set; } +} diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index 690062b7..5747eaba 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -11,6 +11,7 @@ import type { BusGateway, BusGatewayDetail, BusGatewayRow, + DataSourceProvider, DataSourceDetail, DataSourceInspectResult, DataSourceRow, @@ -315,6 +316,8 @@ export interface ApiClient { deleteBusGateway(id: number): Promise; // ——— Data sources ——— + /** What Bitween can connect to, and what each provider accepts. */ + listDataSourceProviders(): Promise; listDataSources(): Promise; searchDataSources(query: { search: string; @@ -327,6 +330,8 @@ export interface ApiClient { adapterId: string; properties: Record; secretProperties: string[]; + /** Broker, Relational, Document, ObjectStore or Http — the provider declares it. */ + kind?: string; }): Promise<{ id: number }>; updateDataSource( id: number, diff --git a/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts b/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts index 20a5f81b..2d1b7eeb 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/dataSources.ts @@ -1,5 +1,6 @@ import type { ApiClient } from "../client"; import type { + DataSourceProvider, DataSourceDetail, DataSourceInspectResult, DataSourceRow, @@ -78,6 +79,11 @@ const toDetail = (raw: RawDataSource): DataSourceDetail => ({ const EVERYTHING = 1_000_000; export const dataSourceMethods = { + async listDataSourceProviders(): Promise { + const providers = await get(`/datasources/Providers`); + return providers ?? []; + }, + async listDataSources(): Promise { const res = await get>( `/datasources?offset=0&limit=${EVERYTHING}`, @@ -109,11 +115,13 @@ export const dataSourceMethods = { adapterId: string; properties: Record; secretProperties: string[]; + kind?: string; }): Promise<{ id: number }> { const id = await post("/datasources", { name: input.name, adapterId: input.adapterId, - kind: "Broker", + // What the provider says it connects to. Only a Broker can back a bus gateway. + kind: input.kind ?? "Broker", properties: input.properties, secretProperties: input.secretProperties, inactive: false, diff --git a/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts b/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts index c7635adb..3042fd7c 100644 --- a/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts +++ b/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts @@ -71,6 +71,8 @@ export const keys = { dataSources: { all: ["data-sources"] as const, + /** The provider catalog, described by the adapters themselves. Rarely changes; cached hard. */ + providers: ["data-sources", "providers"] as const, list: ["data-sources", "list"] as const, search: (params: Record) => ["data-sources", "search", params] as const, detail: (id: number | string) => ["data-sources", "detail", id] as const, diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index de97d428..4a576e4c 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -1176,3 +1176,33 @@ export interface DashboardData { pausedSubscriptions: { id: number; name: string }[]; }; } + +/** + * One connection setting, as the adapter itself declares it. The UI keeps no list of its own — + * see DataSourceProviderCatalog on the server for why. + */ +export interface DataSourceProviderSetting { + name: string; + /** string, number or boolean — enough to pick an input, nothing more. */ + type: "string" | "number" | "boolean"; + hint: string | null; + /** What a new data source starts with. Null means start it empty. */ + default: string | null; + /** When present, the only legal values: rendered as a menu instead of a text box. */ + allowedValues: string[] | null; + secret: boolean; + required: boolean; +} + +export interface DataSourceProvider { + adapterId: string; + label: string; + /** + * What it connects to: Broker, Relational, Document, ObjectStore or Http. A bus gateway can + * only read from a Broker — a provider that holds a database session is a data source too, but + * it has no queue to consume. + */ + kind: string; + description: string | null; + settings: DataSourceProviderSetting[]; +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/SourceDialog.tsx b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/SourceDialog.tsx index 34e3ccae..dffad384 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/SourceDialog.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/SourceDialog.tsx @@ -7,7 +7,7 @@ import { Field, Select, TextInput } from "../../components/ui/forms"; import { Dialog } from "../../components/ui/overlays"; import { keys } from "../../api/queryKeys"; import { ConnectionBadge } from "../data-sources/ConnectionBadge"; -import { providerLabel } from "../data-sources/providers"; +import { providerOf, useDataSourceProviders } from "../data-sources/providers"; /** @@ -31,12 +31,18 @@ export function SourceDialog({ const [endpoint, setEndpoint] = useState(gateway.endpoint ?? ""); const [error, setError] = useState(null); + const providers = useDataSourceProviders(); const sources = useQuery({ queryKey: keys.dataSources.list, queryFn: () => api.listDataSources(), }); - const available = sources.data ?? []; + // Brokers only. A data source can also be a database or an object store held open by a resident + // adapter, and none of those has a queue for a gateway to read — the API refuses them too, so + // this is the same rule stated where the operator can see it rather than a second one. + const available = (sources.data ?? []).filter( + (s) => (s.kind ?? "Broker").toLowerCase() === "broker", + ); const selected = available.find((s) => s.id === dataSourceId); const save = useMutation({ @@ -101,7 +107,7 @@ export function SourceDialog({ {available.length === 0 && ( - No data sources exist yet, so there is no broker to point at.{" "} + No broker data sources exist yet, so there is nothing to point at.{" "} Add one first @@ -118,7 +124,7 @@ export function SourceDialog({ onChange={(e) => setDataSourceId(Number(e.target.value))} options={available.map((s) => ({ value: String(s.id), - label: `${s.name} — ${providerLabel(s.adapterId)}${s.inactive ? " (inactive)" : ""}`, + label: `${s.name} — ${providerOf(providers.data, s.adapterId)?.label ?? s.adapterId}${s.inactive ? " (inactive)" : ""}`, }))} /> diff --git a/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourceNewPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourceNewPage.tsx index 0fc4d827..289ae6b6 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourceNewPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/data-sources/DataSourceNewPage.tsx @@ -3,11 +3,11 @@ import { useNavigate } from "react-router"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { api, ApiRequestError } from "../../api"; import { PageHeader } from "../../components/layout/PageHeader"; -import { Button, FormError } from "../../components/ui/basics"; +import { Button, FormError, LoadingBlock } from "../../components/ui/basics"; import { Field, Select, TextInput } from "../../components/ui/forms"; import { BackLink } from "../../components/ui/BackLink"; import { keys } from "../../api/queryKeys"; -import { BUS_PROVIDERS } from "./providers"; +import { declaredSecrets, initialProperties, useDataSourceProviders } from "./providers"; /** * Creating asks for a name and a provider, and nothing else. @@ -15,25 +15,33 @@ import { BUS_PROVIDERS } from "./providers"; * The connection settings depend on the provider — RabbitMQ wants a virtual host, SQS wants a * region — so asking for them before that is chosen means either the wrong fields or a blank * key/value grid. The provider seeds its own, and the next screen is a form to fill in. + * + * Which providers exist, and what each one starts with, comes from the adapters themselves. */ export function DataSourceNewPage() { const navigate = useNavigate(); const queryClient = useQueryClient(); const [name, setName] = useState(""); - const [adapterId, setAdapterId] = useState(BUS_PROVIDERS[0].id); + const [adapterId, setAdapterId] = useState(null); const [error, setError] = useState(null); - const provider = BUS_PROVIDERS.find((p) => p.id === adapterId)!; + const providers = useDataSourceProviders(); + // Nothing is chosen until the catalog arrives, so the first provider stands in for a choice the + // operator has not made yet. + const provider = providers.data?.find((p) => p.adapterId === adapterId) ?? providers.data?.[0]; const create = useMutation({ - mutationFn: () => - api.createDataSource({ + mutationFn: () => { + if (!provider) throw new Error("No provider chosen."); + return api.createDataSource({ name: name.trim(), - adapterId, - properties: { ...provider.defaults }, - secretProperties: [...provider.secrets], - }), + adapterId: provider.adapterId, + kind: provider.kind, + properties: initialProperties(provider), + secretProperties: declaredSecrets(provider), + }); + }, onSuccess: async ({ id }) => { await queryClient.invalidateQueries({ queryKey: keys.dataSources.all }); navigate(`/data-sources/${id}`); @@ -51,6 +59,21 @@ export function DataSourceNewPage() { create.mutate(); }; + if (providers.isPending) return ; + + // No adapter in the store declares itself a data source provider. Publishing one is the fix, + // and saying so is more use than an empty menu. + if (!provider) + return ( +
+ + + No data source provider adapters are installed. Publish one — bitween.bus.rabbitmq and + bitween.bus.sqs ship with Bitween — and it will appear here. + +
+ ); + return (
@@ -67,12 +90,16 @@ export function DataSourceNewPage() { /> - + setProperty(key, e.target.value)} + // An empty option only where empty is legal: a setting the adapter did + // not mark required can be left for the broker to decide. + options={[ + ...(declared.required ? [] : [{ value: "", label: "—" }]), + ...declared.allowedValues.map((v) => ({ value: v, label: v })), + ]} + /> + ) : secret ? ( setProperty(key, e.target.value)} @@ -546,15 +557,28 @@ export function DataSourcePage() { 0 + ? `${provider?.label ?? "This provider"} also accepts ${unused + .slice(0, 3) + .join(", ")}${unused.length > 3 ? ` and ${unused.length - 3} more` : ""}.` + : "A name that looks like a credential is masked automatically." + } > setNewKey(e.target.value)} onKeyDown={(e) => e.key === "Enter" && addProperty()} /> + {/* Typing is still allowed: an adapter may read more than it declares. */} + + {unused.map((name) => ( +