diff --git a/VariantValidator/modules/vvDBGet.py b/VariantValidator/modules/vvDBGet.py index 05b38af8..e1507bfd 100644 --- a/VariantValidator/modules/vvDBGet.py +++ b/VariantValidator/modules/vvDBGet.py @@ -41,10 +41,6 @@ def _set_cached(key, value): return value -def clear_get_cache(): - DB_GET_CACHE.clear() - - class Mixin(vvDBInit.Mixin): """ Most of the functions in DBGet generate queries for retrieving data @@ -260,15 +256,23 @@ def get_refseq_data_by_refseq_id( ) def get_gene_symbol_from_refseq_id(self, refseq_id): + key = ("gene_symbol_from_refseq_id", refseq_id) + cached = _get_cached(key) + + if cached is not _CACHE_MISS: + return cached + query = ( "SELECT hgncSymbol FROM refSeqGene_loci " "WHERE refSeqGeneID = %s" ) - return self.execute( + result = self.execute( query, (refseq_id,), )[0] + return _set_cached(key, result) + def get_refseq_id_from_lrg_id(self, lrg_id): query = ( "SELECT RefSeqGeneID FROM LRG_RSG_lookup " @@ -375,23 +379,15 @@ def get_lrg_protein_id_from_ref_seq_protein_id(self, rs_p): return _set_cached(key, result) def get_lrg_data_from_lrg_id(self, lrg_id): - key = ("lrg_data_from_lrg_id", lrg_id) - cached = _get_cached(key) - - if cached is not _CACHE_MISS: - return cached - query = ( "SELECT * FROM LRG_RSG_lookup " "WHERE lrgID = %s" ) - result = self.execute( + return self.execute( query, (lrg_id,), ) - return _set_cached(key, result) - def get_transcript_info_for_gene(self, gene_symbol): query = ( "SELECT refSeqID, description, transcriptVariant, " @@ -470,15 +466,23 @@ def get_stable_gene_id_info(self, hgnc_symbol): return _set_cached(key, result) def get_stable_gene_id_from_hgnc_id(self, hgnc_id): + key = ("stable_gene_id_from_hgnc_id", hgnc_id) + cached = _get_cached(key) + + if cached is not _CACHE_MISS: + return cached + query = ( "SELECT * FROM stableGeneIds " "WHERE hgnc_id = %s" ) - return self.execute( + result = self.execute( query, (hgnc_id,), ) + return _set_cached(key, result) + def get_transcripts_from_annotations(self, statement): query = ( "SELECT * FROM transcript_info " @@ -557,18 +561,29 @@ def get_urls(self, dict_out): lrg_data = self.get_lrg_data_from_lrg_id( lrg_id ) - lrg_status = str(lrg_data[4]) - if lrg_status == "public": - report_urls["lrg"] = ( - "http://ftp.ebi.ac.uk/pub/" - f"databases/lrgex/{lrg_id}.xml" - ) - else: - report_urls["lrg"] = ( - "http://ftp.ebi.ac.uk/pub/databases/" - f"lrgex/pending/{lrg_id}.xml" - ) + # LRG identifiers may not have a corresponding lookup + # record in the database. In this case execute() returns + # ["none", "No data"], so only attempt to determine the + # publication status when a valid row has been returned. + if ( + lrg_data + and lrg_data[0] != "none" + and len(lrg_data) > 4 + ): + lrg_status = str(lrg_data[4]) + + if lrg_status == "public": + report_urls["lrg"] = ( + "http://ftp.ebi.ac.uk/pub/" + f"databases/lrgex/{lrg_id}.xml" + ) + else: + report_urls["lrg"] = ( + "http://ftp.ebi.ac.uk/pub/" + "databases/lrgex/pending/" + f"{lrg_id}.xml" + ) # Ensembl. if selected_assembly == "grch37": diff --git a/VariantValidator/modules/vvMixinConverters.py b/VariantValidator/modules/vvMixinConverters.py index 5632eda6..29b8ba75 100644 --- a/VariantValidator/modules/vvMixinConverters.py +++ b/VariantValidator/modules/vvMixinConverters.py @@ -1610,6 +1610,10 @@ def myvm_t_to_g(self, hgvs_c, alt_chr, no_norm_evm, hn, map_dat): hgvs_genomic.posedit.edit.alt = hgvs_genomic.posedit.edit.ref if hgvs_genomic.posedit.edit.type == 'ins' and utilise_gap_code: + + if stored_hgvs_c.posedit.edit.type == "dup": + stored_hgvs_c = hgvs_dup_to_delins(stored_hgvs_c) + try: # Can move ins variants (and in doing so break # mid base == original bases assumption) @@ -1656,7 +1660,7 @@ def myvm_t_to_g(self, hgvs_c, alt_chr, no_norm_evm, hn, map_dat): except AttributeError as e: if "'Dup' object has no attribute 'alt'" in str(e): - logger.error( + logger.exception( "Code triggered previously in very poor alignment so not " "able to fully test, refer to test_inputs.py tests " "test_alt_gapping_bug: hgvs_genomic: %s, " diff --git a/VariantValidator/modules/vvMixinInit.py b/VariantValidator/modules/vvMixinInit.py index 36802662..e682db63 100644 --- a/VariantValidator/modules/vvMixinInit.py +++ b/VariantValidator/modules/vvMixinInit.py @@ -2,6 +2,7 @@ import logging import os +from functools import lru_cache from configparser import ConfigParser import vvhgvs @@ -41,6 +42,56 @@ class InitialisationError(Exception): pass +class CachedSeqFetcher: + """ + LRU cache wrapper for SeqFetcher. + + This class implements the Decorator pattern around SeqFetcher. + Only the most frequently repeated sequence lookup is cached using + functools.lru_cache(); all other methods and attributes are + transparently delegated to the wrapped SeqFetcher. + + The cache exists solely within the current Python process and is + discarded when the process exits. + """ + + def __init__(self, sf): + """ + Wrap an existing SeqFetcher. + + Parameters + ---------- + sf + An instantiated SeqFetcher object. + """ + self._sf = sf + + def __getattr__(self, name): + """ + Delegate all uncached methods and attributes to the wrapped + SeqFetcher. + + Python only calls __getattr__ when an attribute is not found on + CachedSeqFetcher itself. Consequently, only the methods + explicitly implemented below are intercepted and cached; + everything else behaves exactly as if the original SeqFetcher + were being used directly. + """ + return getattr(self._sf, name) + + @lru_cache(maxsize=settings.SEQFETCHER_CACHE_SIZE) + def fetch_seq(self, ac, start_i=None, end_i=None): + """ + Retrieve a sequence or sequence slice. + + The cache key is formed from the accession, start coordinate + and end coordinate. Repeated requests for the same sequence + slice are therefore served directly from memory rather than + performing another SeqRepo lookup. + """ + return self._sf.fetch_seq(ac, start_i, end_i) + + class Mixin: """ Initialise the persistent VariantValidator infrastructure. @@ -76,6 +127,18 @@ def __init__(self): HGVS_SEQREPO_DIR.split('/')[-1] """ + # -------------------------------------------------------------- + # HGVS global configuration + # -------------------------------------------------------------- + + vvhgvs.global_config.uta.pool_max = 25 + vvhgvs.global_config.formatting.max_ref_length = 1000000 + + if settings.vvHGVS_HDP_CACHE: + vvhgvs.global_config.lru_cache.maxsize = settings.vvHGVS_HDP_CACHE_SIZE + else: + vvhgvs.global_config.lru_cache.maxsize = 200 # Default vvHGVS cache size. + # -------------------------------------------------------------- # Configuration # -------------------------------------------------------------- @@ -226,19 +289,11 @@ def __init__(self): self.no_norm_evm = None # -------------------------------------------------------------- - # HGVS global configuration + # HGVS data providers # -------------------------------------------------------------- - vvhgvs.global_config.uta.pool_max = 25 - vvhgvs.global_config.formatting.max_ref_length = 1000000 - - # -------------------------------------------------------------- - # HGVS data provider - # -------------------------------------------------------------- - - self.hdp = vvhgvs.dataproviders.uta.connect( - pooling=True, - ) + # Create the HGVS data provider. + self.hdp = vvhgvs.dataproviders.uta.connect(pooling=True) self.utaSchema = str( self.hdp.data_version() @@ -283,6 +338,16 @@ def __init__(self): self.check_same_thread, ) + # Wrap the SeqFetcher with the LRU cache layer. + # + # CachedSeqFetcher intercepts sequence fetches and caches the returned + # sequence slices. All other methods and attributes are transparently + # delegated to the original SeqFetcher via __getattr__(). + # + # To disable all SeqFetcher caching, simply comment out the line below. + if settings.SEQFETCHER_CACHE: + self.sf = CachedSeqFetcher(self.sf) + # -------------------------------------------------------------- # Persistent alignment-specific normalizers # -------------------------------------------------------------- diff --git a/VariantValidator/settings.py b/VariantValidator/settings.py index 842a4e79..9da85878 100644 --- a/VariantValidator/settings.py +++ b/VariantValidator/settings.py @@ -1,11 +1,197 @@ +""" +=============================================================================== +VariantValidator global settings +=============================================================================== + +This module defines the global configuration used throughout +VariantValidator. + +Configuration values may originate from one of four sources, listed below +in order of precedence. + +Configuration precedence +------------------------ + +1. Runtime overrides + Applications may override the configured logging levels for the current + process using: + + VariantValidator.logger.configure_logging( + console_level=..., + file_level=... + ) + + These overrides affect only the current process and do not modify the + configuration file or the default settings. + +2. Environment variable overrides + Selected settings may be overridden using environment variables, + allowing deployment-specific configuration without modifying the source + code or user configuration file. + +3. VariantValidator configuration file + User configuration is read from: + + ~/.variantvalidator + + unless an alternative configuration file is specified using: + + VARIANTVALIDATOR_TEST_CONFIG + +4. Built-in defaults + Default values defined in this module are used whenever no higher + precedence configuration is supplied. + +=============================================================================== +Cache configuration +=============================================================================== + +VariantValidator uses several optional in-memory caches to improve +performance by reducing repeated database lookups and sequence retrieval +operations. Each cache may be enabled or disabled independently and its +maximum size configured. + +DBGet lookup cache +------------------ +Caches deterministic database lookup results returned by DBGet helper +methods. + +Settings: + + vvDB_GET_CACHE + Enable or disable the DBGet lookup cache. + + vvDB_GET_CACHE_SIZE + Maximum number of cached lookup results. + +Environment overrides: + + VV_DB_GET_CACHE + VV_DB_GET_CACHE_SIZE + + +SeqFetcher cache +---------------- +Caches recently retrieved sequence fragments to avoid repeated sequence +fetch operations. + +Settings: + + SEQFETCHER_CACHE + Enable or disable the SeqFetcher cache. + + SEQFETCHER_CACHE_SIZE + Maximum number of cached sequence fragments. + +Environment overrides: + + SEQFETCHER_CACHE + SEQFETCHER_CACHE_SIZE + + +HGVS data-provider cache +------------------------ +Caches HGVS data-provider (HDP) lookups performed through the local HGVS +interface. + +Settings: + + vvHGVS_HDP_CACHE + Enable or disable the local HGVS HDP cache. + + vvHGVS_HDP_CACHE_SIZE + Maximum number of cached HGVS data-provider lookups. + +Environment overrides: + + VV_HGVS_HDP_CACHE + VV_HGVS_HDP_CACHE_SIZE + +=============================================================================== +Configuration file +=============================================================================== + +VariantValidator reads user configuration from a configuration file +located at: + + ~/.variantvalidator + +This file typically contains: + + • Database connection settings + • Logging configuration + • Other user-specific configuration values + +The configuration file location may be overridden by setting: + + VARIANTVALIDATOR_TEST_CONFIG + +=============================================================================== +Logging configuration +=============================================================================== + +Logging behaviour is configured using the [logging] section of the +VariantValidator configuration file. + +Supported configuration options: + + log + Enable or disable logging globally. + + console + Console logging level. + + file + Log file logging level. + + file_name + Path to the log file. + +If no log file is specified, VariantValidator writes to: + + ~/.vv_errorlog + +Applications may override the configured console and/or file logging +levels for the current process by calling: + + VariantValidator.logger.configure_logging( + console_level=..., + file_level=... + ) + +without modifying the configuration file or the default settings defined +in this module. + +=============================================================================== +""" + import os from configparser import ConfigParser -config = ConfigParser() +# ============================================================================= +# Cache configuration +# +# VariantValidator uses several optional in-memory caches to reduce repeated +# database lookups and sequence retrieval operations. These caches improve +# validation performance for repeated queries while allowing memory usage to be +# tuned for different deployment environments. Each cache can be enabled or +# disabled independently, and cache sizes may be overridden using environment +# variables. +# ============================================================================= +# DBGet lookup cache settings. vvDB_GET_CACHE = True vvDB_GET_CACHE_SIZE = 20000 +# SeqFetcher sequence cache settings. +SEQFETCHER_CACHE = True +SEQFETCHER_CACHE_SIZE = 32768 + +# vvHGVS HDP cache settings. +vvHGVS_HDP_CACHE = True +vvHGVS_HDP_CACHE_SIZE = 1000 + + if "VV_DB_GET_CACHE" in os.environ: vvDB_GET_CACHE = os.environ["VV_DB_GET_CACHE"].lower() in ( "true", @@ -18,6 +204,35 @@ os.environ["VV_DB_GET_CACHE_SIZE"] ) +if "SEQFETCHER_CACHE" in os.environ: + SEQFETCHER_CACHE = os.environ["SEQFETCHER_CACHE"].lower() in ( + "true", + "1", + "yes", + ) + +if "SEQFETCHER_CACHE_SIZE" in os.environ: + SEQFETCHER_CACHE_SIZE = int( + os.environ["SEQFETCHER_CACHE_SIZE"] + ) + +if "vvHGVS_HDP_CACHE" in os.environ: + vvHGVS_HDP_CACHE = os.environ["vvHGVS_HDP_CACHE"].lower() in ( + "true", + "1", + "yes", + ) + +if "vvHGVS_HDP_CACHE_SIZE" in os.environ: + vvHGVS_HDP_CACHE_SIZE = int( + os.environ["vvHGVS_HDP_CACHE_SIZE"] + ) + + +# ---------------------------------------- +# READ VV CONFIG FILE +# ---------------------------------------- +config = ConfigParser() def get_config_dir(): if 'VARIANTVALIDATOR_TEST_CONFIG' in os.environ: return os.environ['VARIANTVALIDATOR_TEST_CONFIG'] @@ -27,7 +242,6 @@ def get_config_dir(): '.variantvalidator' ) -# Read config config.read(get_config_dir()) @@ -39,6 +253,7 @@ def get_config_dir(): '.vv_errorlog' ) + # ---------------------------------------- # FILE LOCATION (SAFE) # ---------------------------------------- @@ -50,6 +265,7 @@ def get_config_dir(): LOG_FILE = file_name if file_name else DEFAULT_LOG LOG_FILE = os.path.abspath(os.path.expanduser(LOG_FILE)) + # ---------------------------------------- # ENSURE LOG DIRECTORY EXISTS # ---------------------------------------- @@ -57,11 +273,13 @@ def get_config_dir(): if log_dir: os.makedirs(log_dir, exist_ok=True) + # ---------------------------------------- # GLOBAL LOGGING SWITCH # ---------------------------------------- logging_enabled = config.get('logging', 'log', fallback='true').lower() not in ('false', '0', 'no', 'off') + # ---------------------------------------- # LOG LEVELS # ---------------------------------------- @@ -73,6 +291,7 @@ def get_config_dir(): CONSOLE_LEVEL = 'CRITICAL' FILE_LEVEL = 'CRITICAL' + # ---------------------------------------- # LOGGING CONFIG # ---------------------------------------- diff --git a/tests/variantvalidator/test_DBGet.py b/tests/variantvalidator/test_DBGet.py index f7f30447..d9176661 100644 --- a/tests/variantvalidator/test_DBGet.py +++ b/tests/variantvalidator/test_DBGet.py @@ -9,7 +9,6 @@ _CACHE_MISS, _get_cached, _set_cached, - clear_get_cache, ) @@ -18,17 +17,6 @@ def make_db(): return db -@pytest.fixture(autouse=True) -def reset_db_get_cache(monkeypatch): - clear_get_cache() - monkeypatch.setattr(settings, "vvDB_GET_CACHE", True) - monkeypatch.setattr(settings, "vvDB_GET_CACHE_SIZE", 10000) - - yield - - clear_get_cache() - - def test_execute_fetchone_success(): db = make_db() @@ -337,25 +325,16 @@ def test_cache_set_and_get(): def test_cache_disabled(monkeypatch): monkeypatch.setattr(settings, "vvDB_GET_CACHE", False) + before = len(DB_GET_CACHE) + key = ("test", "A") _set_cached(key, "value") - assert key not in DB_GET_CACHE + assert len(DB_GET_CACHE) == before assert _get_cached(key) is _CACHE_MISS -def test_cache_clear(): - _set_cached(("test", "A"), "A") - _set_cached(("test", "B"), "B") - - assert len(DB_GET_CACHE) == 2 - - clear_get_cache() - - assert len(DB_GET_CACHE) == 0 - - def test_cache_respects_maximum_size(monkeypatch): monkeypatch.setattr(settings, "vvDB_GET_CACHE_SIZE", 2) @@ -421,12 +400,6 @@ def test_cache_hit_refreshes_lru_order(monkeypatch): ["LRG_1p1"], "LRG_1p1", ), - ( - "get_lrg_data_from_lrg_id", - "LRG_1", - ["data"], - ["data"], - ), ( "get_stable_gene_id_info", "GENE1", @@ -451,6 +424,15 @@ def test_cached_getters_only_query_database_once( db.execute.assert_called_once() +def test_get_lrg_data_from_lrg_id_queries_database_each_time(): + db = make_db() + db.execute = MagicMock(return_value=["data"]) + + assert db.get_lrg_data_from_lrg_id("LRG_1") == ["data"] + assert db.get_lrg_data_from_lrg_id("LRG_1") == ["data"] + + assert db.execute.call_count == 2 + def test_cached_getter_queries_again_when_cache_disabled( monkeypatch, diff --git a/tests/variantvalidator/test_inputs.py b/tests/variantvalidator/test_inputs.py index 5ecb3f43..3564ffba 100644 --- a/tests/variantvalidator/test_inputs.py +++ b/tests/variantvalidator/test_inputs.py @@ -31354,6 +31354,14 @@ def test_regression_start_lost_translation(self): 'ProteinTranslationError: Unable to generate protein variant description due to the reference sequence ' 'missing an accepted start codon.'] + def test_regression_pkd1(self): + results = self.vv.validate('NM_000296.4:c.4781dup', 'GRCh38', 'all', + liftover_level=True).format_as_dict(test=True) + assert "NM_000296.4:c.4781dup" in results.keys() + results = self.vv.validate('NM_000296.4:c.11308_11309dup', 'GRCh38', 'all', + liftover_level=True).format_as_dict(test=True) + assert "NM_000296.4:c.11308_11309dup" in results.keys() + # Copyright (C) 2016-2026 VariantValidator Contributors # This file is part of VariantValidator and is distributed under the diff --git a/tests/variantvalidator/test_settings.py b/tests/variantvalidator/test_settings.py index fe6f4f70..c41cf18f 100644 --- a/tests/variantvalidator/test_settings.py +++ b/tests/variantvalidator/test_settings.py @@ -15,6 +15,65 @@ CONFIG_DIR = settings.get_config_dir() +def test_environment_cache_overrides(monkeypatch): + """ + Verify that cache settings are correctly overridden by + environment variables. + """ + monkeypatch.setenv("VV_DB_GET_CACHE", "true") + monkeypatch.setenv("VV_DB_GET_CACHE_SIZE", "12345") + + monkeypatch.setenv("SEQFETCHER_CACHE", "false") + monkeypatch.setenv("SEQFETCHER_CACHE_SIZE", "54321") + + monkeypatch.setenv("vvHGVS_HDP_CACHE", "false") + monkeypatch.setenv("vvHGVS_HDP_CACHE_SIZE", "999") + + importlib.reload(settings) + + assert settings.vvDB_GET_CACHE is True + assert settings.vvDB_GET_CACHE_SIZE == 12345 + + assert settings.SEQFETCHER_CACHE is False + assert settings.SEQFETCHER_CACHE_SIZE == 54321 + + assert settings.vvHGVS_HDP_CACHE is False + assert settings.vvHGVS_HDP_CACHE_SIZE == 999 + +def test_environment_test_config_override(monkeypatch): + """ + Verify that the VariantValidator configuration file location can + be overridden using VARIANTVALIDATOR_TEST_CONFIG. + """ + monkeypatch.setenv( + "VARIANTVALIDATOR_TEST_CONFIG", + "/tmp/test_variantvalidator.ini", + ) + + importlib.reload(settings) + + assert ( + settings.get_config_dir() + == "/tmp/test_variantvalidator.ini" + ) + +def test_get_config_dir_default(monkeypatch): + """ + Verify the default VariantValidator configuration directory is + returned when no environment override is present. + """ + monkeypatch.delenv( + "VARIANTVALIDATOR_TEST_CONFIG", + raising=False, + ) + + importlib.reload(settings) + + assert settings.get_config_dir().endswith( + ".variantvalidator" + ) + + class TestSettings(TestCase): def test_config_dir_exists(self): assert os.path.exists(CONFIG_DIR) diff --git a/tests/variantvalidator/test_vvMixinInit.py b/tests/variantvalidator/test_vvMixinInit.py index 5dc5e4d2..bd642d77 100644 --- a/tests/variantvalidator/test_vvMixinInit.py +++ b/tests/variantvalidator/test_vvMixinInit.py @@ -7,7 +7,9 @@ Mixin, InitialisationError, ) -from VariantValidator import version +from VariantValidator import version, settings +from VariantValidator.validator import Validator + class TestVVMixinInit(TestCase): @@ -328,6 +330,45 @@ def test_environment_variables( os.environ["UTA_DB_URL"], ) +def test_seqfetcher_cache_enabled(monkeypatch): + monkeypatch.setattr(settings, "SEQFETCHER_CACHE", True) + monkeypatch.setattr( + settings, + "SEQFETCHER_CACHE_SIZE", + 32768, + ) + + validator = Validator() + + assert ( + validator.sf.fetch_seq.cache_info().maxsize + == settings.SEQFETCHER_CACHE_SIZE + ) + + +def test_hdp_cache_disabled(monkeypatch): + monkeypatch.setattr(settings, "vvHGVS_HDP_CACHE", False) + + validator = Validator() + + assert validator.hdp.get_seq.cache_info().maxsize == 200 + + +def test_hdp_cache_enabled(monkeypatch): + monkeypatch.setattr(settings, "vvHGVS_HDP_CACHE", True) + monkeypatch.setattr( + settings, + "vvHGVS_HDP_CACHE_SIZE", + 1000, + ) + + validator = Validator() + + assert ( + validator.hdp.get_seq.cache_info().maxsize + == settings.vvHGVS_HDP_CACHE_SIZE + ) + # Copyright (C) 2016-2026 VariantValidator Contributors # This file is part of VariantValidator and is distributed under the # GNU Affero General Public License, version 3 or (at your option) any