diff --git a/.changeset/org-resolver-cache.md b/.changeset/org-resolver-cache.md new file mode 100644 index 00000000000..45fd6e03ef2 --- /dev/null +++ b/.changeset/org-resolver-cache.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#added Durable caching of OrgResolver owner->orgID mappings (backed by Postgres, `CRE.Linking.DurableCacheEnabled`, enabled by default). diff --git a/core/config/cre_config.go b/core/config/cre_config.go index 229bd4648a8..7d6119b2a39 100644 --- a/core/config/cre_config.go +++ b/core/config/cre_config.go @@ -40,4 +40,6 @@ type CRELinking interface { TLSEnabled() bool // RequestTimeout bounds each organization lookup against the linking service. RequestTimeout() time.Duration + // DurableCacheEnabled turns on durable caching of owner->orgID mappings (backed by Postgres). + DurableCacheEnabled() bool } diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index e2292d1ff40..667f086d31b 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -997,6 +997,8 @@ URL = "" # Default TLSEnabled = true # Default # RequestTimeout bounds each organization lookup against the linking service. RequestTimeout = '2s' # Default +# DurableCacheEnabled turns on durable Postgres-backed caching of owner->orgID mappings. +DurableCacheEnabled = true # Default # Billing holds settings for connecting to the billing service. [Billing] diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 75629c5610c..c2949270b50 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -2058,6 +2058,8 @@ type LinkingConfig struct { URL *string `toml:",omitempty"` TLSEnabled *bool `toml:",omitempty"` RequestTimeout *commonconfig.Duration `toml:",omitempty"` + // DurableCacheEnabled turns on durable Postgres-backed caching of owner->orgID mappings. + DurableCacheEnabled *bool `toml:",omitempty"` } func (c *CreConfig) setFrom(f *CreConfig) { @@ -2105,6 +2107,9 @@ func (c *CreConfig) setFrom(f *CreConfig) { if v := f.Linking.RequestTimeout; v != nil { c.Linking.RequestTimeout = v } + if v := f.Linking.DurableCacheEnabled; v != nil { + c.Linking.DurableCacheEnabled = v + } } if f.DebugMode != nil { @@ -2158,6 +2163,10 @@ func (l *LinkingConfig) ValidateConfig() error { } else if l.RequestTimeout.Duration() <= 0 { return configutils.ErrInvalid{Name: "RequestTimeout", Value: l.RequestTimeout.String(), Msg: "must be positive"} } + if l.DurableCacheEnabled == nil { + val := true + l.DurableCacheEnabled = &val + } return nil } diff --git a/core/services/chainlink/config_cre.go b/core/services/chainlink/config_cre.go index f66c83b8e46..c4063d4c162 100644 --- a/core/services/chainlink/config_cre.go +++ b/core/services/chainlink/config_cre.go @@ -80,9 +80,10 @@ func (c *creConfig) EnableDKGRecipient() bool { } type linkingConfig struct { - url string - tlsEnabled bool - requestTimeout time.Duration + url string + tlsEnabled bool + requestTimeout time.Duration + durableCacheEnabled bool } func (l *linkingConfig) URL() string { @@ -97,9 +98,13 @@ func (l *linkingConfig) RequestTimeout() time.Duration { return l.requestTimeout } +func (l *linkingConfig) DurableCacheEnabled() bool { + return l.durableCacheEnabled +} + func (c *creConfig) Linking() config.CRELinking { if c.c.Linking == nil { - return &linkingConfig{url: "", tlsEnabled: true, requestTimeout: defaultLinkingRequestTimeout} + return &linkingConfig{url: "", tlsEnabled: true, requestTimeout: defaultLinkingRequestTimeout, durableCacheEnabled: true} } url := "" @@ -117,7 +122,12 @@ func (c *creConfig) Linking() config.CRELinking { requestTimeout = c.c.Linking.RequestTimeout.Duration() } - return &linkingConfig{url: url, tlsEnabled: tlsEnabled, requestTimeout: requestTimeout} + durableCacheEnabled := true // default + if c.c.Linking.DurableCacheEnabled != nil { + durableCacheEnabled = *c.c.Linking.DurableCacheEnabled + } + + return &linkingConfig{url: url, tlsEnabled: tlsEnabled, requestTimeout: requestTimeout, durableCacheEnabled: durableCacheEnabled} } type confidentialRelayConfig struct { diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 2d0eca02159..1664fa75f51 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -615,9 +615,10 @@ func TestConfig_Marshal(t *testing.T) { URL: new("https://workflow.fetcher.url"), }, Linking: &toml.LinkingConfig{ - URL: new(""), - TLSEnabled: new(true), - RequestTimeout: commoncfg.MustNewDuration(2 * time.Second), + URL: new(""), + TLSEnabled: new(true), + RequestTimeout: commoncfg.MustNewDuration(2 * time.Second), + DurableCacheEnabled: new(true), }, ConfidentialRelay: &toml.ConfidentialRelayConfig{ Enabled: new(bool), diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 44af3e15812..203d47dd527 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -407,6 +407,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index bc185b6b2e9..571d9cf3778 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -446,6 +446,7 @@ URL = 'https://workflow.fetcher.url' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 8a705a5cf14..e4fd45296fa 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -407,6 +407,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 70c036e5604..07e707e660a 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -240,13 +240,12 @@ func (s *Services) newSubservices( if cfg.CRE().Linking().URL() != "" { lggr.Debugw("Creating OrgResolver") - inner, ierr := newOrgResolver(cfg, capCfg, opts, lggr) + resolver, ierr := newOrgResolver(cfg, capCfg, opts, ds, lggr) if ierr != nil { return nil, fmt.Errorf("could not create org resolver: %w", ierr) } - fallbackResolver := orgresolver.NewOrgResolverWithFallback(inner, lggr) - s.OrgResolver = fallbackResolver - srvs = append(srvs, fallbackResolver) + s.OrgResolver = resolver + srvs = append(srvs, resolver) } else { lggr.Warn("Skipping orgResolver, no linking service configured") } @@ -631,6 +630,7 @@ func newOrgResolver( cfg Config, capCfg config.Capabilities, opts Opts, + ds sqlutil.DataSource, lggr logger.Logger, ) (orgresolver.OrgResolver, error) { var wrChainDetails chainselectors.ChainDetails @@ -661,7 +661,22 @@ func newOrgResolver( return nil, fmt.Errorf("failed to create org resolver: %w", err) } - return resolver, nil + var cache orgresolver.Cache + if cfg.CRE().Linking().DurableCacheEnabled() { + cache = NewOrgResolverStore(ds) + } else { + cache = orgresolver.NewInMemoryCache() + } + + cachingResolver, err := orgresolver.NewCachingResolver(resolver, orgresolver.CachingResolverConfig{ + Cache: cache, + Meter: opts.Meter, + }, lggr) + if err != nil { + return nil, fmt.Errorf("failed to create caching org resolver: %w", err) + } + + return cachingResolver, nil } func newBillingClient(lggr logger.Logger, cfg Config, opts Opts) (metering.BillingClient, error) { diff --git a/core/services/cre/org_resolver_store.go b/core/services/cre/org_resolver_store.go new file mode 100644 index 00000000000..a94e787cad4 --- /dev/null +++ b/core/services/cre/org_resolver_store.go @@ -0,0 +1,57 @@ +package cre + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" +) + +// orgResolverCacheTable is the durable owner->orgID mapping table backing the +// OrgResolver cache. See migration 0307_org_resolver_cache.sql. +const orgResolverCacheTable = "cre.org_resolver_cache" + +// orgResolverStore is a Postgres-backed implementation of orgresolver.Cache. +type orgResolverStore struct { + ds sqlutil.DataSource +} + +// NewOrgResolverStore creates a durable cache store for the OrgResolver. +func NewOrgResolverStore(ds sqlutil.DataSource) *orgResolverStore { + return &orgResolverStore{ds: ds} +} + +// Get returns the cached entry for owner. ok is false if no entry exists. +func (s *orgResolverStore) Get(ctx context.Context, owner string) (orgresolver.CacheEntry, bool, error) { + const q = `SELECT org_id, updated_at FROM ` + orgResolverCacheTable + ` WHERE workflow_owner = $1` + var row struct { + OrgID string `db:"org_id"` + UpdatedAt time.Time `db:"updated_at"` + } + if err := s.ds.GetContext(ctx, &row, q, owner); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return orgresolver.CacheEntry{}, false, nil + } + return orgresolver.CacheEntry{}, false, fmt.Errorf("failed to get cached org for owner %s: %w", owner, err) + } + return orgresolver.CacheEntry{OrgID: row.OrgID, RefreshedAt: row.UpdatedAt}, true, nil +} + +// Set stores or updates the mapping for owner. +// This will soon be optimized by batching, once Linking Service supports batch queries. +func (s *orgResolverStore) Set(ctx context.Context, owner string, entry orgresolver.CacheEntry) error { + const q = ` +INSERT INTO ` + orgResolverCacheTable + ` (workflow_owner, org_id, updated_at) +VALUES ($1, $2, $3) +ON CONFLICT (workflow_owner) DO UPDATE SET org_id = EXCLUDED.org_id, updated_at = EXCLUDED.updated_at` + if _, err := s.ds.ExecContext(ctx, q, owner, entry.OrgID, entry.RefreshedAt); err != nil { + return fmt.Errorf("failed to upsert org for owner %s: %w", owner, err) + } + return nil +} + +var _ orgresolver.Cache = (*orgResolverStore)(nil) diff --git a/core/services/cre/org_resolver_store_test.go b/core/services/cre/org_resolver_store_test.go new file mode 100644 index 00000000000..b2418f86b12 --- /dev/null +++ b/core/services/cre/org_resolver_store_test.go @@ -0,0 +1,48 @@ +package cre + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" +) + +func Test_OrgResolverStore_GetSet(t *testing.T) { + t.Parallel() + db := pgtest.NewSqlxDB(t) + store := NewOrgResolverStore(db) + ctx := context.Background() + + _, ok, err := store.Get(ctx, "owner-1") + require.NoError(t, err) + assert.False(t, ok) + + entry := orgresolver.CacheEntry{OrgID: "org-1", RefreshedAt: time.Now().UTC().Truncate(time.Microsecond)} + require.NoError(t, store.Set(ctx, "owner-1", entry)) + + got, ok, err := store.Get(ctx, "owner-1") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, entry.OrgID, got.OrgID) + assert.WithinDuration(t, entry.RefreshedAt, got.RefreshedAt, time.Second) +} + +func Test_OrgResolverStore_Set_UpsertsOnOrgIDChange(t *testing.T) { + t.Parallel() + db := pgtest.NewSqlxDB(t) + store := NewOrgResolverStore(db) + ctx := context.Background() + + require.NoError(t, store.Set(ctx, "owner-1", orgresolver.CacheEntry{OrgID: "org-1", RefreshedAt: time.Now()})) + require.NoError(t, store.Set(ctx, "owner-1", orgresolver.CacheEntry{OrgID: "org-2", RefreshedAt: time.Now()})) + + var orgID string + require.NoError(t, db.GetContext(ctx, &orgID, + `SELECT org_id FROM cre.org_resolver_cache WHERE workflow_owner = $1`, "owner-1")) + assert.Equal(t, "org-2", orgID) +} diff --git a/core/store/migrate/migrations/0307_org_resolver_cache.sql b/core/store/migrate/migrations/0307_org_resolver_cache.sql new file mode 100644 index 00000000000..a5a42dad91c --- /dev/null +++ b/core/store/migrate/migrations/0307_org_resolver_cache.sql @@ -0,0 +1,12 @@ +-- +goose Up + +CREATE TABLE IF NOT EXISTS cre.org_resolver_cache ( + workflow_owner TEXT NOT NULL PRIMARY KEY, + org_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- +goose Down + +DROP TABLE IF EXISTS cre.org_resolver_cache; diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 44af3e15812..203d47dd527 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -407,6 +407,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 06cef4f6f84..56bf14e4a69 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -425,6 +425,7 @@ URL = 'https://workflow.fetcher.url' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index b1089686c29..76eb3496580 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -407,6 +407,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 8e6c05fb4c6..b83348ca63f 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2766,6 +2766,7 @@ URL is override URL for the workflow fetcher service. URL = "" # Default TLSEnabled = true # Default RequestTimeout = '2s' # Default +DurableCacheEnabled = true # Default ``` @@ -2787,6 +2788,12 @@ RequestTimeout = '2s' # Default ``` RequestTimeout bounds each organization lookup against the linking service. +### DurableCacheEnabled +```toml +DurableCacheEnabled = true # Default +``` +DurableCacheEnabled turns on durable Postgres-backed caching of owner->orgID mappings. + ## Billing ```toml [Billing] diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 807c1a14fcf..9f3974669c1 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -554,6 +554,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index cc45943e52d..4200761e10e 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -419,6 +419,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 2c651d0a0cb..52527323753 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -480,6 +480,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 78761600502..f00a068214c 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -463,6 +463,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 3c868382f92..c8e7e97cd80 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -463,6 +463,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index aa7389b0620..07dfc6dcbae 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -463,6 +463,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 0109df7b7fc..087f2e648c0 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -565,6 +565,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index 061f537f325..5683aab3cde 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -448,6 +448,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 364a068aee7..b9afc66f0a5 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -459,6 +459,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 19508dd1590..2bf9a67b3bc 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -460,6 +460,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 1d3bda09504..175cfb53846 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -442,6 +442,7 @@ URL = '' URL = '' TLSEnabled = true RequestTimeout = '2s' +DurableCacheEnabled = true [CRE.ConfidentialRelay] Enabled = false