diff --git a/pkg/settings/cresettings/README.md b/pkg/settings/cresettings/README.md index 0ce55544d0..5ebb8528e3 100644 --- a/pkg/settings/cresettings/README.md +++ b/pkg/settings/cresettings/README.md @@ -76,6 +76,8 @@ flowchart GatewayConfidentialRelayPerNodeRate[\GatewayConfidentialRelayPerNodeRate/]:::rate GatewayHTTPActionMtlsRequestRate[\GatewayHTTPActionMtlsRequestRate/]:::rate GatewayHTTPActionMtlsConcurrencyLimit([GatewayHTTPActionMtlsConcurrencyLimit]):::resource + GatewayHTTPActionOutboundConcurrencyLimit([GatewayHTTPActionOutboundConcurrencyLimit]):::resource + GatewayHTTPActionOutboundPerNodeConcurrencyLimit([GatewayHTTPActionOutboundPerNodeConcurrencyLimit]):::resource end %% TODO unused %% PerOrg.ZeroBalancePruningTimeout diff --git a/pkg/settings/cresettings/defaults.json b/pkg/settings/cresettings/defaults.json index 4e0505e8e1..41b38973ca 100644 --- a/pkg/settings/cresettings/defaults.json +++ b/pkg/settings/cresettings/defaults.json @@ -26,6 +26,8 @@ "GatewayConfidentialRelayPerNodeRate": "10rps:10", "GatewayHTTPActionMtlsRequestRate": "every30s:0", "GatewayHTTPActionMtlsConcurrencyLimit": "50", + "GatewayHTTPActionOutboundConcurrencyLimit": "875", + "GatewayHTTPActionOutboundPerNodeConcurrencyLimit": "175", "TriggerRegistrationStatusUpdateTimeout": "0s", "BaseTriggerRetryInterval": "30s", "BaseTriggerMaxRetries": "20", diff --git a/pkg/settings/cresettings/defaults.toml b/pkg/settings/cresettings/defaults.toml index 3004605698..706874ea47 100644 --- a/pkg/settings/cresettings/defaults.toml +++ b/pkg/settings/cresettings/defaults.toml @@ -25,6 +25,8 @@ GatewayConfidentialRelayGlobalRate = '50rps:10' GatewayConfidentialRelayPerNodeRate = '10rps:10' GatewayHTTPActionMtlsRequestRate = 'every30s:0' GatewayHTTPActionMtlsConcurrencyLimit = '50' +GatewayHTTPActionOutboundConcurrencyLimit = '875' +GatewayHTTPActionOutboundPerNodeConcurrencyLimit = '175' TriggerRegistrationStatusUpdateTimeout = '0s' BaseTriggerRetryInterval = '30s' BaseTriggerMaxRetries = '20' diff --git a/pkg/settings/cresettings/settings.go b/pkg/settings/cresettings/settings.go index e07cfb558a..07c09ddcd4 100644 --- a/pkg/settings/cresettings/settings.go +++ b/pkg/settings/cresettings/settings.go @@ -79,20 +79,22 @@ var Default = Schema{ VaultIncludeInvalidPendingItemsEnabled: Bool(false), VaultPendingQueueStallThreshold: Int(0), // Deprecated: feature flag has been retired; behavior is now always enabled. - VaultSignedResponseRequestIDEnabled: Bool(false), - VaultZoneBWorkflowGetSecretsRestrictEnabled: Bool(false), - GatewayHTTPGlobalRate: Rate(rate.Limit(500), 500), - GatewayHTTPPerNodeRate: Rate(rate.Limit(100), 100), - GatewayConfidentialRelayGlobalRate: Rate(rate.Limit(50), 10), - GatewayConfidentialRelayPerNodeRate: Rate(rate.Limit(10), 10), - GatewayHTTPActionMtlsRequestRate: Rate(rate.Every(30*time.Second), 0), - GatewayHTTPActionMtlsConcurrencyLimit: Int(50), - TriggerRegistrationStatusUpdateTimeout: Duration(0 * time.Second), - BaseTriggerRetryInterval: Duration(30 * time.Second), - BaseTriggerMaxRetries: Int(20), - BaseTriggerPruneAge: Duration(24 * time.Hour), - BaseTriggerMaxSendsPerTick: Int(20), - WASMPollOneoffSubscriptionLimit: Int(128), + VaultSignedResponseRequestIDEnabled: Bool(false), + VaultZoneBWorkflowGetSecretsRestrictEnabled: Bool(false), + GatewayHTTPGlobalRate: Rate(rate.Limit(500), 500), + GatewayHTTPPerNodeRate: Rate(rate.Limit(100), 100), + GatewayConfidentialRelayGlobalRate: Rate(rate.Limit(50), 10), + GatewayConfidentialRelayPerNodeRate: Rate(rate.Limit(10), 10), + GatewayHTTPActionMtlsRequestRate: Rate(rate.Every(30*time.Second), 0), + GatewayHTTPActionMtlsConcurrencyLimit: Int(50), + GatewayHTTPActionOutboundConcurrencyLimit: Int(875), + GatewayHTTPActionOutboundPerNodeConcurrencyLimit: Int(175), + TriggerRegistrationStatusUpdateTimeout: Duration(0 * time.Second), + BaseTriggerRetryInterval: Duration(30 * time.Second), + BaseTriggerMaxRetries: Int(20), + BaseTriggerPruneAge: Duration(24 * time.Hour), + BaseTriggerMaxSendsPerTick: Int(20), + WASMPollOneoffSubscriptionLimit: Int(128), // DANGER(cedric): Be extremely careful changing these vault limits below as they act as a default value // used by the Vault OCR plugin -- changing these values could cause issues with the plugin during an image @@ -392,15 +394,27 @@ type Schema struct { VaultIncludeInvalidPendingItemsEnabled Setting[bool] VaultPendingQueueStallThreshold Setting[int] `unit:"{observation}"` // Deprecated: feature flag has been retired; behavior is now always enabled. - VaultSignedResponseRequestIDEnabled Setting[bool] - VaultZoneBWorkflowGetSecretsRestrictEnabled Setting[bool] - GatewayHTTPGlobalRate Setting[config.Rate] - GatewayHTTPPerNodeRate Setting[config.Rate] - GatewayConfidentialRelayGlobalRate Setting[config.Rate] - GatewayConfidentialRelayPerNodeRate Setting[config.Rate] - GatewayHTTPActionMtlsRequestRate Setting[config.Rate] - GatewayHTTPActionMtlsConcurrencyLimit Setting[int] `unit:"{request}"` - TriggerRegistrationStatusUpdateTimeout Setting[time.Duration] + VaultSignedResponseRequestIDEnabled Setting[bool] + VaultZoneBWorkflowGetSecretsRestrictEnabled Setting[bool] + GatewayHTTPGlobalRate Setting[config.Rate] + GatewayHTTPPerNodeRate Setting[config.Rate] + GatewayConfidentialRelayGlobalRate Setting[config.Rate] + GatewayConfidentialRelayPerNodeRate Setting[config.Rate] + GatewayHTTPActionMtlsRequestRate Setting[config.Rate] + GatewayHTTPActionMtlsConcurrencyLimit Setting[int] `unit:"{request}"` + // GatewayHTTPActionOutboundConcurrencyLimit bounds the number of outbound HTTP action + // requests the gateway will have in flight at once, across all nodes. Sized to + // GatewayHTTPGlobalRate's ceiling (500rps burst) times observed p99.9 outbound latency + // (~1.75s in production-mainnet), so this never binds tighter than the rate limit already + // permits: 500 * 1.75 ~= 875. + GatewayHTTPActionOutboundConcurrencyLimit Setting[int] `unit:"{request}"` + // GatewayHTTPActionOutboundPerNodeConcurrencyLimit bounds the number of outbound HTTP + // action requests the gateway will have in flight for a single node, so one node cannot + // occupy every slot in GatewayHTTPActionOutboundConcurrencyLimit. Sized the same way as + // GatewayHTTPActionOutboundConcurrencyLimit, against GatewayHTTPPerNodeRate's ceiling + // (100rps burst): 100 * 1.75 ~= 175. + GatewayHTTPActionOutboundPerNodeConcurrencyLimit Setting[int] `unit:"{request}"` + TriggerRegistrationStatusUpdateTimeout Setting[time.Duration] BaseTriggerRetryInterval Setting[time.Duration] BaseTriggerMaxRetries Setting[int] `unit:"{attempt}"`