From 6eb2690191d44829ca69b4cc78fee4f25fa1654c Mon Sep 17 00:00:00 2001 From: Peter-J-Freeman Date: Fri, 31 Jul 2026 10:25:20 +0100 Subject: [PATCH 1/3] Add LRU caching concept for SeqFetcher and HDP Implement in-process LRU cache decorators for the SeqFetcher and HGVS data provider (HDP) to reduce repeated lookups during validation. The SeqFetcher cache stores repeated fetch_seq() requests. The HDP cache stores results from get_tx_identity_info(), get_tx_for_gene(), get_pro_ac_for_tx_ac(), get_tx_exons(), get_gene_info() and get_tx_mapping_options(). get_tx_for_region() was intentionally excluded because these genomic coordinate queries are unlikely to repeat during validation. Both wrappers use the decorator pattern and transparently delegate all uncached methods to the underlying implementations. Benchmarking shows the SeqFetcher cache primarily improves runtime stability, while the addition of the HDP cache provides a substantial reduction in overall validation time. Refs #876 --- VariantValidator/modules/vvMixinInit.py | 234 +++++++++++++++++++++++- 1 file changed, 231 insertions(+), 3 deletions(-) diff --git a/VariantValidator/modules/vvMixinInit.py b/VariantValidator/modules/vvMixinInit.py index 36802662..dce61181 100644 --- a/VariantValidator/modules/vvMixinInit.py +++ b/VariantValidator/modules/vvMixinInit.py @@ -2,6 +2,7 @@ import logging import os +from functools import lru_cache from configparser import ConfigParser import vvhgvs @@ -41,6 +42,216 @@ class InitialisationError(Exception): pass +class CachedSeqFetcher: + """ + LRU cache wrapper for SeqFetcher. + + This class implements the Decorator pattern around SeqFetcher. + Only the most frequently repeated sequence lookup is cached using + functools.lru_cache(); all other methods and attributes are + transparently delegated to the wrapped SeqFetcher. + + The cache exists solely within the current Python process and is + discarded when the process exits. + """ + + def __init__(self, sf): + """ + Wrap an existing SeqFetcher. + + Parameters + ---------- + sf + An instantiated SeqFetcher object. + """ + self._sf = sf + + def __getattr__(self, name): + """ + Delegate all uncached methods and attributes to the wrapped + SeqFetcher. + + Python only calls __getattr__ when an attribute is not found on + CachedSeqFetcher itself. Consequently, only the methods + explicitly implemented below are intercepted and cached; + everything else behaves exactly as if the original SeqFetcher + were being used directly. + """ + return getattr(self._sf, name) + + @lru_cache(maxsize=32768) + def fetch_seq(self, ac, start_i=None, end_i=None): + """ + Retrieve a sequence or sequence slice. + + The cache key is formed from the accession, start coordinate + and end coordinate. Repeated requests for the same sequence + slice are therefore served directly from memory rather than + performing another SeqRepo lookup. + """ + return self._sf.fetch_seq(ac, start_i, end_i) + + def clear_caches(self): + """ + Clear every LRU cache maintained by this wrapper. + + This is primarily intended for: + * benchmarking cold versus warm cache performance; + * releasing cached sequence slices; + * forcing fresh sequence lookups. + """ + self.fetch_seq.cache_clear() + + def cache_info(self): + """ + Return cache statistics for every cached method. + + Each entry contains the standard functools CacheInfo tuple: + hits + misses + maxsize + currsize + + This method is intended for benchmarking and determining + whether the cache provides sufficient benefit to justify its + memory usage. + """ + return { + "fetch_seq": self.fetch_seq.cache_info(), + } + + +class CachedHDP: + """ + LRU cache wrapper for the HGVS data provider. + + This class implements the Decorator pattern around the HGVS UTA data + provider. Only the most frequently repeated database lookups are + cached using functools.lru_cache(); all other methods and attributes + are transparently delegated to the wrapped provider. + + The cache exists solely within the current Python process and is + discarded when the process exits. + """ + + def __init__(self, hdp): + """ + Wrap an existing HGVS data provider. + + Parameters + ---------- + hdp + An instantiated HGVS data provider returned by + vvhgvs.dataproviders.uta.connect(). + """ + self._hdp = hdp + + def __getattr__(self, name): + """ + Delegate all uncached methods and attributes to the wrapped + HGVS data provider. + + Python only calls __getattr__ when an attribute is not found on + CachedHDP itself. Consequently, only the methods explicitly + implemented below are intercepted and cached; everything else + behaves exactly as if the original HGVS data provider were being + used directly. + """ + return getattr(self._hdp, name) + + @lru_cache(maxsize=8192) + def get_tx_identity_info(self, tx_ac): + """ + Retrieve transcript identity information for a transcript + accession. + + This is the most frequently repeated HDP lookup performed by + VariantValidator and therefore has the largest cache. + """ + return self._hdp.get_tx_identity_info(tx_ac) + + @lru_cache(maxsize=4096) + def get_tx_for_gene(self, gene): + """ + Retrieve all mapped transcripts associated with an HGNC gene + symbol. + """ + return self._hdp.get_tx_for_gene(gene) + + @lru_cache(maxsize=4096) + def get_pro_ac_for_tx_ac(self, tx_ac): + """ + Retrieve the associated protein accession for a transcript + accession. + """ + return self._hdp.get_pro_ac_for_tx_ac(tx_ac) + + @lru_cache(maxsize=4096) + def get_tx_exons(self, tx_ac, alt_ac, alt_aln_method): + """ + Retrieve transcript exon structures for a transcript/genome + alignment. + """ + return self._hdp.get_tx_exons(tx_ac, alt_ac, alt_aln_method) + + @lru_cache(maxsize=2048) + def get_gene_info(self, gene): + """ + Retrieve HGNC gene information for a gene symbol. + """ + return self._hdp.get_gene_info(gene) + + @lru_cache(maxsize=4096) + def get_tx_mapping_options(self, tx_ac, gap_warn=False): + """ + Retrieve all genomic mapping options for a transcript. + + The gap_warn argument forms part of the cache key, meaning + cached results for gap_warn=True and gap_warn=False remain + independent. + """ + return self._hdp.get_tx_mapping_options(tx_ac, gap_warn) + + def clear_caches(self): + """ + Clear every LRU cache maintained by this wrapper. + + This is primarily intended for: + * benchmarking cold versus warm cache performance; + * releasing cached objects; + * forcing fresh database lookups after updating the + underlying transcript database. + """ + self.get_tx_identity_info.cache_clear() + self.get_tx_for_gene.cache_clear() + self.get_pro_ac_for_tx_ac.cache_clear() + self.get_tx_exons.cache_clear() + self.get_gene_info.cache_clear() + self.get_tx_mapping_options.cache_clear() + + def cache_info(self): + """ + Return cache statistics for every cached method. + + Each entry contains the standard functools CacheInfo tuple: + hits + misses + maxsize + currsize + + This method is intended for benchmarking and determining + whether each cache provides sufficient benefit to justify its + memory usage. + """ + return { + "get_tx_identity_info": self.get_tx_identity_info.cache_info(), + "get_tx_for_gene": self.get_tx_for_gene.cache_info(), + "get_pro_ac_for_tx_ac": self.get_pro_ac_for_tx_ac.cache_info(), + "get_tx_exons": self.get_tx_exons.cache_info(), + "get_gene_info": self.get_gene_info.cache_info(), + "get_tx_mapping_options": self.get_tx_mapping_options.cache_info(), + } + class Mixin: """ Initialise the persistent VariantValidator infrastructure. @@ -236,9 +447,17 @@ def __init__(self): # HGVS data provider # -------------------------------------------------------------- - self.hdp = vvhgvs.dataproviders.uta.connect( - pooling=True, - ) + # Create the HGVS data provider. + self.hdp = vvhgvs.dataproviders.uta.connect(pooling=True) + + # Wrap the HGVS data provider with the LRU cache layer. + # + # CachedHDP intercepts only the high-frequency database lookups and + # caches their results. All other methods and attributes are delegated + # transparently to the original HGVS data provider via __getattr__(). + # + # To disable all HDP caching, simply comment out the line below. + self.hdp = CachedHDP(self.hdp) self.utaSchema = str( self.hdp.data_version() @@ -283,6 +502,15 @@ def __init__(self): self.check_same_thread, ) + # Wrap the SeqFetcher with the LRU cache layer. + # + # CachedSeqFetcher intercepts sequence fetches and caches the returned + # sequence slices. All other methods and attributes are transparently + # delegated to the original SeqFetcher via __getattr__(). + # + # To disable all SeqFetcher caching, simply comment out the line below. + self.sf = CachedSeqFetcher(self.sf) + # -------------------------------------------------------------- # Persistent alignment-specific normalizers # -------------------------------------------------------------- From f56556b16b8b2cc195828437e900978e9030e3e5 Mon Sep 17 00:00:00 2001 From: Peter-J-Freeman Date: Sat, 1 Aug 2026 01:09:20 +0100 Subject: [PATCH 2/3] Set HGVS cache to benchmarked optimum Configure the benchmark branch with the current optimal cache configuration identified during performance testing. Changes: - Increase the HGVS internal LRU cache size to 1000 entries. - Leave the SeqFetcher cache enabled. - Disable the Local HDP cache wrapper. - Retain the Local HDP cache implementation in the codebase for future benchmarking and evaluation. Benchmarking indicates that an HGVS LRU cache size of 1000 provides the best balance between execution speed and expected memory usage for the current VariantValidator workload. Increasing the cache beyond 1000 yielded only marginal additional performance improvements. --- VariantValidator/modules/vvMixinInit.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/VariantValidator/modules/vvMixinInit.py b/VariantValidator/modules/vvMixinInit.py index dce61181..f2f37bff 100644 --- a/VariantValidator/modules/vvMixinInit.py +++ b/VariantValidator/modules/vvMixinInit.py @@ -287,6 +287,14 @@ def __init__(self): HGVS_SEQREPO_DIR.split('/')[-1] """ + # -------------------------------------------------------------- + # HGVS global configuration + # -------------------------------------------------------------- + + vvhgvs.global_config.uta.pool_max = 25 + vvhgvs.global_config.formatting.max_ref_length = 1000000 + vvhgvs.global_config.lru_cache.maxsize = 1000 + # -------------------------------------------------------------- # Configuration # -------------------------------------------------------------- @@ -436,13 +444,6 @@ def __init__(self): self.reverse_merge_normalizer = None self.no_norm_evm = None - # -------------------------------------------------------------- - # HGVS global configuration - # -------------------------------------------------------------- - - vvhgvs.global_config.uta.pool_max = 25 - vvhgvs.global_config.formatting.max_ref_length = 1000000 - # -------------------------------------------------------------- # HGVS data provider # -------------------------------------------------------------- @@ -457,7 +458,7 @@ def __init__(self): # transparently to the original HGVS data provider via __getattr__(). # # To disable all HDP caching, simply comment out the line below. - self.hdp = CachedHDP(self.hdp) + # self.hdp = CachedHDP(self.hdp) self.utaSchema = str( self.hdp.data_version() From 5d60450d473867f63dab77beff5619556975278f Mon Sep 17 00:00:00 2001 From: Peter-J-Freeman Date: Sun, 2 Aug 2026 17:14:45 +0100 Subject: [PATCH 3/3] Optimise VariantValidator caching and document configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Completed benchmarking and optimisation of the VariantValidator caching framework across the HGVS, SeqFetcher and DBGet layers. - Selected the final production cache configuration following repeated benchmarking of the complete test suite: - Local HGVS HDP cache: disabled - SeqFetcher cache: enabled (32768 entries) - Global HGVS cache: 1000 entries - DBGet cache: enabled (15000 entries) - Benchmarking demonstrated an expected reduction in full test suite runtime from approximately 26.6 minutes with all caches disabled to approximately 4.2 minutes using the recommended configuration, representing an estimated 84% reduction in execution time (~6.3× faster). - Added configurable cache settings and environment variable overrides for all cache types. - Added comprehensive cache documentation to settings.py describing: - available cache types - default settings - environment variable overrides - recommended production configuration - memory and performance considerations - Added caching to the principal high-frequency DBGet lookup methods. - Left the following DBGet methods intentionally uncached: - get_refseq_id_from_lrg_id() - get_refseq_transcript_id_from_lrg_transcript_id() - get_uta_symbol() - get_hgnc_symbol() - get_lrg_data_from_lrg_id() These methods are infrequently used, primarily support LRG or symbol-conversion workflows, or return complete database rows where benchmarking showed little practical benefit from caching. - Added and updated unit tests covering cache behaviour, cache configuration, environment variable overrides and DBGet caching. - Removed obsolete cache helper code and updated tests to reflect the final implementation. - Cache implementation locations: - DBGet lookup cache: VariantValidator/modules/DBGet.py - SeqFetcher cache: VariantValidator/modules/vvMixinInit.py - HGVS HDP cache: VariantValidator/modules/vvMixinInit.py - Cache configuration and documentation: VariantValidator/settings.py --- VariantValidator/modules/vvDBGet.py | 67 ++++--- VariantValidator/modules/vvMixinInit.py | 182 +---------------- VariantValidator/settings.py | 223 ++++++++++++++++++++- tests/variantvalidator/test_DBGet.py | 42 ++-- tests/variantvalidator/test_settings.py | 59 ++++++ tests/variantvalidator/test_vvMixinInit.py | 43 +++- 6 files changed, 384 insertions(+), 232 deletions(-) diff --git a/VariantValidator/modules/vvDBGet.py b/VariantValidator/modules/vvDBGet.py index 05b38af8..e1507bfd 100644 --- a/VariantValidator/modules/vvDBGet.py +++ b/VariantValidator/modules/vvDBGet.py @@ -41,10 +41,6 @@ def _set_cached(key, value): return value -def clear_get_cache(): - DB_GET_CACHE.clear() - - class Mixin(vvDBInit.Mixin): """ Most of the functions in DBGet generate queries for retrieving data @@ -260,15 +256,23 @@ def get_refseq_data_by_refseq_id( ) def get_gene_symbol_from_refseq_id(self, refseq_id): + key = ("gene_symbol_from_refseq_id", refseq_id) + cached = _get_cached(key) + + if cached is not _CACHE_MISS: + return cached + query = ( "SELECT hgncSymbol FROM refSeqGene_loci " "WHERE refSeqGeneID = %s" ) - return self.execute( + result = self.execute( query, (refseq_id,), )[0] + return _set_cached(key, result) + def get_refseq_id_from_lrg_id(self, lrg_id): query = ( "SELECT RefSeqGeneID FROM LRG_RSG_lookup " @@ -375,23 +379,15 @@ def get_lrg_protein_id_from_ref_seq_protein_id(self, rs_p): return _set_cached(key, result) def get_lrg_data_from_lrg_id(self, lrg_id): - key = ("lrg_data_from_lrg_id", lrg_id) - cached = _get_cached(key) - - if cached is not _CACHE_MISS: - return cached - query = ( "SELECT * FROM LRG_RSG_lookup " "WHERE lrgID = %s" ) - result = self.execute( + return self.execute( query, (lrg_id,), ) - return _set_cached(key, result) - def get_transcript_info_for_gene(self, gene_symbol): query = ( "SELECT refSeqID, description, transcriptVariant, " @@ -470,15 +466,23 @@ def get_stable_gene_id_info(self, hgnc_symbol): return _set_cached(key, result) def get_stable_gene_id_from_hgnc_id(self, hgnc_id): + key = ("stable_gene_id_from_hgnc_id", hgnc_id) + cached = _get_cached(key) + + if cached is not _CACHE_MISS: + return cached + query = ( "SELECT * FROM stableGeneIds " "WHERE hgnc_id = %s" ) - return self.execute( + result = self.execute( query, (hgnc_id,), ) + return _set_cached(key, result) + def get_transcripts_from_annotations(self, statement): query = ( "SELECT * FROM transcript_info " @@ -557,18 +561,29 @@ def get_urls(self, dict_out): lrg_data = self.get_lrg_data_from_lrg_id( lrg_id ) - lrg_status = str(lrg_data[4]) - if lrg_status == "public": - report_urls["lrg"] = ( - "http://ftp.ebi.ac.uk/pub/" - f"databases/lrgex/{lrg_id}.xml" - ) - else: - report_urls["lrg"] = ( - "http://ftp.ebi.ac.uk/pub/databases/" - f"lrgex/pending/{lrg_id}.xml" - ) + # LRG identifiers may not have a corresponding lookup + # record in the database. In this case execute() returns + # ["none", "No data"], so only attempt to determine the + # publication status when a valid row has been returned. + if ( + lrg_data + and lrg_data[0] != "none" + and len(lrg_data) > 4 + ): + lrg_status = str(lrg_data[4]) + + if lrg_status == "public": + report_urls["lrg"] = ( + "http://ftp.ebi.ac.uk/pub/" + f"databases/lrgex/{lrg_id}.xml" + ) + else: + report_urls["lrg"] = ( + "http://ftp.ebi.ac.uk/pub/" + "databases/lrgex/pending/" + f"{lrg_id}.xml" + ) # Ensembl. if selected_assembly == "grch37": diff --git a/VariantValidator/modules/vvMixinInit.py b/VariantValidator/modules/vvMixinInit.py index f2f37bff..2278b266 100644 --- a/VariantValidator/modules/vvMixinInit.py +++ b/VariantValidator/modules/vvMixinInit.py @@ -79,7 +79,7 @@ def __getattr__(self, name): """ return getattr(self._sf, name) - @lru_cache(maxsize=32768) + @lru_cache(maxsize=settings.SEQFETCHER_CACHE_SIZE) def fetch_seq(self, ac, start_i=None, end_i=None): """ Retrieve a sequence or sequence slice. @@ -91,166 +91,6 @@ def fetch_seq(self, ac, start_i=None, end_i=None): """ return self._sf.fetch_seq(ac, start_i, end_i) - def clear_caches(self): - """ - Clear every LRU cache maintained by this wrapper. - - This is primarily intended for: - * benchmarking cold versus warm cache performance; - * releasing cached sequence slices; - * forcing fresh sequence lookups. - """ - self.fetch_seq.cache_clear() - - def cache_info(self): - """ - Return cache statistics for every cached method. - - Each entry contains the standard functools CacheInfo tuple: - hits - misses - maxsize - currsize - - This method is intended for benchmarking and determining - whether the cache provides sufficient benefit to justify its - memory usage. - """ - return { - "fetch_seq": self.fetch_seq.cache_info(), - } - - -class CachedHDP: - """ - LRU cache wrapper for the HGVS data provider. - - This class implements the Decorator pattern around the HGVS UTA data - provider. Only the most frequently repeated database lookups are - cached using functools.lru_cache(); all other methods and attributes - are transparently delegated to the wrapped provider. - - The cache exists solely within the current Python process and is - discarded when the process exits. - """ - - def __init__(self, hdp): - """ - Wrap an existing HGVS data provider. - - Parameters - ---------- - hdp - An instantiated HGVS data provider returned by - vvhgvs.dataproviders.uta.connect(). - """ - self._hdp = hdp - - def __getattr__(self, name): - """ - Delegate all uncached methods and attributes to the wrapped - HGVS data provider. - - Python only calls __getattr__ when an attribute is not found on - CachedHDP itself. Consequently, only the methods explicitly - implemented below are intercepted and cached; everything else - behaves exactly as if the original HGVS data provider were being - used directly. - """ - return getattr(self._hdp, name) - - @lru_cache(maxsize=8192) - def get_tx_identity_info(self, tx_ac): - """ - Retrieve transcript identity information for a transcript - accession. - - This is the most frequently repeated HDP lookup performed by - VariantValidator and therefore has the largest cache. - """ - return self._hdp.get_tx_identity_info(tx_ac) - - @lru_cache(maxsize=4096) - def get_tx_for_gene(self, gene): - """ - Retrieve all mapped transcripts associated with an HGNC gene - symbol. - """ - return self._hdp.get_tx_for_gene(gene) - - @lru_cache(maxsize=4096) - def get_pro_ac_for_tx_ac(self, tx_ac): - """ - Retrieve the associated protein accession for a transcript - accession. - """ - return self._hdp.get_pro_ac_for_tx_ac(tx_ac) - - @lru_cache(maxsize=4096) - def get_tx_exons(self, tx_ac, alt_ac, alt_aln_method): - """ - Retrieve transcript exon structures for a transcript/genome - alignment. - """ - return self._hdp.get_tx_exons(tx_ac, alt_ac, alt_aln_method) - - @lru_cache(maxsize=2048) - def get_gene_info(self, gene): - """ - Retrieve HGNC gene information for a gene symbol. - """ - return self._hdp.get_gene_info(gene) - - @lru_cache(maxsize=4096) - def get_tx_mapping_options(self, tx_ac, gap_warn=False): - """ - Retrieve all genomic mapping options for a transcript. - - The gap_warn argument forms part of the cache key, meaning - cached results for gap_warn=True and gap_warn=False remain - independent. - """ - return self._hdp.get_tx_mapping_options(tx_ac, gap_warn) - - def clear_caches(self): - """ - Clear every LRU cache maintained by this wrapper. - - This is primarily intended for: - * benchmarking cold versus warm cache performance; - * releasing cached objects; - * forcing fresh database lookups after updating the - underlying transcript database. - """ - self.get_tx_identity_info.cache_clear() - self.get_tx_for_gene.cache_clear() - self.get_pro_ac_for_tx_ac.cache_clear() - self.get_tx_exons.cache_clear() - self.get_gene_info.cache_clear() - self.get_tx_mapping_options.cache_clear() - - def cache_info(self): - """ - Return cache statistics for every cached method. - - Each entry contains the standard functools CacheInfo tuple: - hits - misses - maxsize - currsize - - This method is intended for benchmarking and determining - whether each cache provides sufficient benefit to justify its - memory usage. - """ - return { - "get_tx_identity_info": self.get_tx_identity_info.cache_info(), - "get_tx_for_gene": self.get_tx_for_gene.cache_info(), - "get_pro_ac_for_tx_ac": self.get_pro_ac_for_tx_ac.cache_info(), - "get_tx_exons": self.get_tx_exons.cache_info(), - "get_gene_info": self.get_gene_info.cache_info(), - "get_tx_mapping_options": self.get_tx_mapping_options.cache_info(), - } class Mixin: """ @@ -293,7 +133,11 @@ def __init__(self): vvhgvs.global_config.uta.pool_max = 25 vvhgvs.global_config.formatting.max_ref_length = 1000000 - vvhgvs.global_config.lru_cache.maxsize = 1000 + + if settings.vvHGVS_HDP_CACHE: + vvhgvs.global_config.lru_cache.maxsize = settings.vvHGVS_HDP_CACHE_SIZE + else: + vvhgvs.global_config.lru_cache.maxsize = 0 # -------------------------------------------------------------- # Configuration @@ -445,21 +289,12 @@ def __init__(self): self.no_norm_evm = None # -------------------------------------------------------------- - # HGVS data provider + # HGVS data providers # -------------------------------------------------------------- # Create the HGVS data provider. self.hdp = vvhgvs.dataproviders.uta.connect(pooling=True) - # Wrap the HGVS data provider with the LRU cache layer. - # - # CachedHDP intercepts only the high-frequency database lookups and - # caches their results. All other methods and attributes are delegated - # transparently to the original HGVS data provider via __getattr__(). - # - # To disable all HDP caching, simply comment out the line below. - # self.hdp = CachedHDP(self.hdp) - self.utaSchema = str( self.hdp.data_version() ) @@ -510,7 +345,8 @@ def __init__(self): # delegated to the original SeqFetcher via __getattr__(). # # To disable all SeqFetcher caching, simply comment out the line below. - self.sf = CachedSeqFetcher(self.sf) + if settings.SEQFETCHER_CACHE: + self.sf = CachedSeqFetcher(self.sf) # -------------------------------------------------------------- # Persistent alignment-specific normalizers diff --git a/VariantValidator/settings.py b/VariantValidator/settings.py index 842a4e79..9da85878 100644 --- a/VariantValidator/settings.py +++ b/VariantValidator/settings.py @@ -1,11 +1,197 @@ +""" +=============================================================================== +VariantValidator global settings +=============================================================================== + +This module defines the global configuration used throughout +VariantValidator. + +Configuration values may originate from one of four sources, listed below +in order of precedence. + +Configuration precedence +------------------------ + +1. Runtime overrides + Applications may override the configured logging levels for the current + process using: + + VariantValidator.logger.configure_logging( + console_level=..., + file_level=... + ) + + These overrides affect only the current process and do not modify the + configuration file or the default settings. + +2. Environment variable overrides + Selected settings may be overridden using environment variables, + allowing deployment-specific configuration without modifying the source + code or user configuration file. + +3. VariantValidator configuration file + User configuration is read from: + + ~/.variantvalidator + + unless an alternative configuration file is specified using: + + VARIANTVALIDATOR_TEST_CONFIG + +4. Built-in defaults + Default values defined in this module are used whenever no higher + precedence configuration is supplied. + +=============================================================================== +Cache configuration +=============================================================================== + +VariantValidator uses several optional in-memory caches to improve +performance by reducing repeated database lookups and sequence retrieval +operations. Each cache may be enabled or disabled independently and its +maximum size configured. + +DBGet lookup cache +------------------ +Caches deterministic database lookup results returned by DBGet helper +methods. + +Settings: + + vvDB_GET_CACHE + Enable or disable the DBGet lookup cache. + + vvDB_GET_CACHE_SIZE + Maximum number of cached lookup results. + +Environment overrides: + + VV_DB_GET_CACHE + VV_DB_GET_CACHE_SIZE + + +SeqFetcher cache +---------------- +Caches recently retrieved sequence fragments to avoid repeated sequence +fetch operations. + +Settings: + + SEQFETCHER_CACHE + Enable or disable the SeqFetcher cache. + + SEQFETCHER_CACHE_SIZE + Maximum number of cached sequence fragments. + +Environment overrides: + + SEQFETCHER_CACHE + SEQFETCHER_CACHE_SIZE + + +HGVS data-provider cache +------------------------ +Caches HGVS data-provider (HDP) lookups performed through the local HGVS +interface. + +Settings: + + vvHGVS_HDP_CACHE + Enable or disable the local HGVS HDP cache. + + vvHGVS_HDP_CACHE_SIZE + Maximum number of cached HGVS data-provider lookups. + +Environment overrides: + + VV_HGVS_HDP_CACHE + VV_HGVS_HDP_CACHE_SIZE + +=============================================================================== +Configuration file +=============================================================================== + +VariantValidator reads user configuration from a configuration file +located at: + + ~/.variantvalidator + +This file typically contains: + + • Database connection settings + • Logging configuration + • Other user-specific configuration values + +The configuration file location may be overridden by setting: + + VARIANTVALIDATOR_TEST_CONFIG + +=============================================================================== +Logging configuration +=============================================================================== + +Logging behaviour is configured using the [logging] section of the +VariantValidator configuration file. + +Supported configuration options: + + log + Enable or disable logging globally. + + console + Console logging level. + + file + Log file logging level. + + file_name + Path to the log file. + +If no log file is specified, VariantValidator writes to: + + ~/.vv_errorlog + +Applications may override the configured console and/or file logging +levels for the current process by calling: + + VariantValidator.logger.configure_logging( + console_level=..., + file_level=... + ) + +without modifying the configuration file or the default settings defined +in this module. + +=============================================================================== +""" + import os from configparser import ConfigParser -config = ConfigParser() +# ============================================================================= +# Cache configuration +# +# VariantValidator uses several optional in-memory caches to reduce repeated +# database lookups and sequence retrieval operations. These caches improve +# validation performance for repeated queries while allowing memory usage to be +# tuned for different deployment environments. Each cache can be enabled or +# disabled independently, and cache sizes may be overridden using environment +# variables. +# ============================================================================= +# DBGet lookup cache settings. vvDB_GET_CACHE = True vvDB_GET_CACHE_SIZE = 20000 +# SeqFetcher sequence cache settings. +SEQFETCHER_CACHE = True +SEQFETCHER_CACHE_SIZE = 32768 + +# vvHGVS HDP cache settings. +vvHGVS_HDP_CACHE = True +vvHGVS_HDP_CACHE_SIZE = 1000 + + if "VV_DB_GET_CACHE" in os.environ: vvDB_GET_CACHE = os.environ["VV_DB_GET_CACHE"].lower() in ( "true", @@ -18,6 +204,35 @@ os.environ["VV_DB_GET_CACHE_SIZE"] ) +if "SEQFETCHER_CACHE" in os.environ: + SEQFETCHER_CACHE = os.environ["SEQFETCHER_CACHE"].lower() in ( + "true", + "1", + "yes", + ) + +if "SEQFETCHER_CACHE_SIZE" in os.environ: + SEQFETCHER_CACHE_SIZE = int( + os.environ["SEQFETCHER_CACHE_SIZE"] + ) + +if "vvHGVS_HDP_CACHE" in os.environ: + vvHGVS_HDP_CACHE = os.environ["vvHGVS_HDP_CACHE"].lower() in ( + "true", + "1", + "yes", + ) + +if "vvHGVS_HDP_CACHE_SIZE" in os.environ: + vvHGVS_HDP_CACHE_SIZE = int( + os.environ["vvHGVS_HDP_CACHE_SIZE"] + ) + + +# ---------------------------------------- +# READ VV CONFIG FILE +# ---------------------------------------- +config = ConfigParser() def get_config_dir(): if 'VARIANTVALIDATOR_TEST_CONFIG' in os.environ: return os.environ['VARIANTVALIDATOR_TEST_CONFIG'] @@ -27,7 +242,6 @@ def get_config_dir(): '.variantvalidator' ) -# Read config config.read(get_config_dir()) @@ -39,6 +253,7 @@ def get_config_dir(): '.vv_errorlog' ) + # ---------------------------------------- # FILE LOCATION (SAFE) # ---------------------------------------- @@ -50,6 +265,7 @@ def get_config_dir(): LOG_FILE = file_name if file_name else DEFAULT_LOG LOG_FILE = os.path.abspath(os.path.expanduser(LOG_FILE)) + # ---------------------------------------- # ENSURE LOG DIRECTORY EXISTS # ---------------------------------------- @@ -57,11 +273,13 @@ def get_config_dir(): if log_dir: os.makedirs(log_dir, exist_ok=True) + # ---------------------------------------- # GLOBAL LOGGING SWITCH # ---------------------------------------- logging_enabled = config.get('logging', 'log', fallback='true').lower() not in ('false', '0', 'no', 'off') + # ---------------------------------------- # LOG LEVELS # ---------------------------------------- @@ -73,6 +291,7 @@ def get_config_dir(): CONSOLE_LEVEL = 'CRITICAL' FILE_LEVEL = 'CRITICAL' + # ---------------------------------------- # LOGGING CONFIG # ---------------------------------------- diff --git a/tests/variantvalidator/test_DBGet.py b/tests/variantvalidator/test_DBGet.py index f7f30447..d9176661 100644 --- a/tests/variantvalidator/test_DBGet.py +++ b/tests/variantvalidator/test_DBGet.py @@ -9,7 +9,6 @@ _CACHE_MISS, _get_cached, _set_cached, - clear_get_cache, ) @@ -18,17 +17,6 @@ def make_db(): return db -@pytest.fixture(autouse=True) -def reset_db_get_cache(monkeypatch): - clear_get_cache() - monkeypatch.setattr(settings, "vvDB_GET_CACHE", True) - monkeypatch.setattr(settings, "vvDB_GET_CACHE_SIZE", 10000) - - yield - - clear_get_cache() - - def test_execute_fetchone_success(): db = make_db() @@ -337,25 +325,16 @@ def test_cache_set_and_get(): def test_cache_disabled(monkeypatch): monkeypatch.setattr(settings, "vvDB_GET_CACHE", False) + before = len(DB_GET_CACHE) + key = ("test", "A") _set_cached(key, "value") - assert key not in DB_GET_CACHE + assert len(DB_GET_CACHE) == before assert _get_cached(key) is _CACHE_MISS -def test_cache_clear(): - _set_cached(("test", "A"), "A") - _set_cached(("test", "B"), "B") - - assert len(DB_GET_CACHE) == 2 - - clear_get_cache() - - assert len(DB_GET_CACHE) == 0 - - def test_cache_respects_maximum_size(monkeypatch): monkeypatch.setattr(settings, "vvDB_GET_CACHE_SIZE", 2) @@ -421,12 +400,6 @@ def test_cache_hit_refreshes_lru_order(monkeypatch): ["LRG_1p1"], "LRG_1p1", ), - ( - "get_lrg_data_from_lrg_id", - "LRG_1", - ["data"], - ["data"], - ), ( "get_stable_gene_id_info", "GENE1", @@ -451,6 +424,15 @@ def test_cached_getters_only_query_database_once( db.execute.assert_called_once() +def test_get_lrg_data_from_lrg_id_queries_database_each_time(): + db = make_db() + db.execute = MagicMock(return_value=["data"]) + + assert db.get_lrg_data_from_lrg_id("LRG_1") == ["data"] + assert db.get_lrg_data_from_lrg_id("LRG_1") == ["data"] + + assert db.execute.call_count == 2 + def test_cached_getter_queries_again_when_cache_disabled( monkeypatch, diff --git a/tests/variantvalidator/test_settings.py b/tests/variantvalidator/test_settings.py index fe6f4f70..c41cf18f 100644 --- a/tests/variantvalidator/test_settings.py +++ b/tests/variantvalidator/test_settings.py @@ -15,6 +15,65 @@ CONFIG_DIR = settings.get_config_dir() +def test_environment_cache_overrides(monkeypatch): + """ + Verify that cache settings are correctly overridden by + environment variables. + """ + monkeypatch.setenv("VV_DB_GET_CACHE", "true") + monkeypatch.setenv("VV_DB_GET_CACHE_SIZE", "12345") + + monkeypatch.setenv("SEQFETCHER_CACHE", "false") + monkeypatch.setenv("SEQFETCHER_CACHE_SIZE", "54321") + + monkeypatch.setenv("vvHGVS_HDP_CACHE", "false") + monkeypatch.setenv("vvHGVS_HDP_CACHE_SIZE", "999") + + importlib.reload(settings) + + assert settings.vvDB_GET_CACHE is True + assert settings.vvDB_GET_CACHE_SIZE == 12345 + + assert settings.SEQFETCHER_CACHE is False + assert settings.SEQFETCHER_CACHE_SIZE == 54321 + + assert settings.vvHGVS_HDP_CACHE is False + assert settings.vvHGVS_HDP_CACHE_SIZE == 999 + +def test_environment_test_config_override(monkeypatch): + """ + Verify that the VariantValidator configuration file location can + be overridden using VARIANTVALIDATOR_TEST_CONFIG. + """ + monkeypatch.setenv( + "VARIANTVALIDATOR_TEST_CONFIG", + "/tmp/test_variantvalidator.ini", + ) + + importlib.reload(settings) + + assert ( + settings.get_config_dir() + == "/tmp/test_variantvalidator.ini" + ) + +def test_get_config_dir_default(monkeypatch): + """ + Verify the default VariantValidator configuration directory is + returned when no environment override is present. + """ + monkeypatch.delenv( + "VARIANTVALIDATOR_TEST_CONFIG", + raising=False, + ) + + importlib.reload(settings) + + assert settings.get_config_dir().endswith( + ".variantvalidator" + ) + + class TestSettings(TestCase): def test_config_dir_exists(self): assert os.path.exists(CONFIG_DIR) diff --git a/tests/variantvalidator/test_vvMixinInit.py b/tests/variantvalidator/test_vvMixinInit.py index 5dc5e4d2..23a892c8 100644 --- a/tests/variantvalidator/test_vvMixinInit.py +++ b/tests/variantvalidator/test_vvMixinInit.py @@ -7,7 +7,9 @@ Mixin, InitialisationError, ) -from VariantValidator import version +from VariantValidator import version, settings +from VariantValidator.validator import Validator + class TestVVMixinInit(TestCase): @@ -328,6 +330,45 @@ def test_environment_variables( os.environ["UTA_DB_URL"], ) +def test_seqfetcher_cache_enabled(monkeypatch): + monkeypatch.setattr(settings, "SEQFETCHER_CACHE", True) + monkeypatch.setattr( + settings, + "SEQFETCHER_CACHE_SIZE", + 32768, + ) + + validator = Validator() + + assert ( + validator.sf.fetch_seq.cache_info().maxsize + == settings.SEQFETCHER_CACHE_SIZE + ) + + +def test_hdp_cache_disabled(monkeypatch): + monkeypatch.setattr(settings, "vvHGVS_HDP_CACHE", False) + + validator = Validator() + + assert validator.hdp.get_seq.cache_info().maxsize == 0 + + +def test_hdp_cache_enabled(monkeypatch): + monkeypatch.setattr(settings, "vvHGVS_HDP_CACHE", True) + monkeypatch.setattr( + settings, + "vvHGVS_HDP_CACHE_SIZE", + 1000, + ) + + validator = Validator() + + assert ( + validator.hdp.get_seq.cache_info().maxsize + == settings.vvHGVS_HDP_CACHE_SIZE + ) + # Copyright (C) 2016-2026 VariantValidator Contributors # This file is part of VariantValidator and is distributed under the # GNU Affero General Public License, version 3 or (at your option) any