Optimise VariantValidator caching and document configuration

VariantValidator/modules/vvDBGet.py

@@ -41,10 +41,6 @@ def _set_cached(key, value):
     return value
 
 
-def clear_get_cache():
-    DB_GET_CACHE.clear()
-
-
 class Mixin(vvDBInit.Mixin):
     """
     Most of the functions in DBGet generate queries for retrieving data
@@ -260,15 +256,23 @@ def get_refseq_data_by_refseq_id(
         )
 
     def get_gene_symbol_from_refseq_id(self, refseq_id):
+        key = ("gene_symbol_from_refseq_id", refseq_id)
+        cached = _get_cached(key)
+
+        if cached is not _CACHE_MISS:
+            return cached
+
         query = (
             "SELECT hgncSymbol FROM refSeqGene_loci "
             "WHERE refSeqGeneID = %s"
         )
-        return self.execute(
+        result = self.execute(
             query,
             (refseq_id,),
         )[0]
 
+        return _set_cached(key, result)
+
     def get_refseq_id_from_lrg_id(self, lrg_id):
         query = (
             "SELECT RefSeqGeneID FROM LRG_RSG_lookup "
@@ -375,23 +379,15 @@ def get_lrg_protein_id_from_ref_seq_protein_id(self, rs_p):
         return _set_cached(key, result)
 
     def get_lrg_data_from_lrg_id(self, lrg_id):
-        key = ("lrg_data_from_lrg_id", lrg_id)
-        cached = _get_cached(key)
-
-        if cached is not _CACHE_MISS:
-            return cached
-
         query = (
             "SELECT * FROM LRG_RSG_lookup "
             "WHERE lrgID = %s"
         )
-        result = self.execute(
+        return self.execute(
             query,
             (lrg_id,),
         )
 
-        return _set_cached(key, result)
-
     def get_transcript_info_for_gene(self, gene_symbol):
         query = (
             "SELECT refSeqID, description, transcriptVariant, "
@@ -470,15 +466,23 @@ def get_stable_gene_id_info(self, hgnc_symbol):
         return _set_cached(key, result)
 
     def get_stable_gene_id_from_hgnc_id(self, hgnc_id):
+        key = ("stable_gene_id_from_hgnc_id", hgnc_id)
+        cached = _get_cached(key)
+
+        if cached is not _CACHE_MISS:
+            return cached
+
         query = (
             "SELECT * FROM stableGeneIds "
             "WHERE hgnc_id = %s"
         )
-        return self.execute(
+        result = self.execute(
             query,
             (hgnc_id,),
         )
 
+        return _set_cached(key, result)
+
     def get_transcripts_from_annotations(self, statement):
         query = (
             "SELECT * FROM transcript_info "
@@ -557,18 +561,29 @@ def get_urls(self, dict_out):
             lrg_data = self.get_lrg_data_from_lrg_id(
                 lrg_id
             )
-            lrg_status = str(lrg_data[4])
 
-            if lrg_status == "public":
-                report_urls["lrg"] = (
-                    "http://ftp.ebi.ac.uk/pub/"
-                    f"databases/lrgex/{lrg_id}.xml"
-                )
-            else:
-                report_urls["lrg"] = (
-                    "http://ftp.ebi.ac.uk/pub/databases/"
-                    f"lrgex/pending/{lrg_id}.xml"
-                )
+            # LRG identifiers may not have a corresponding lookup
+            # record in the database. In this case execute() returns
+            # ["none", "No data"], so only attempt to determine the
+            # publication status when a valid row has been returned.
+            if (
+                lrg_data
+                and lrg_data[0] != "none"
+                and len(lrg_data) > 4
+            ):
+                lrg_status = str(lrg_data[4])
+
+                if lrg_status == "public":
+                    report_urls["lrg"] = (
+                        "http://ftp.ebi.ac.uk/pub/"
+                        f"databases/lrgex/{lrg_id}.xml"
+                    )
+                else:
+                    report_urls["lrg"] = (
+                        "http://ftp.ebi.ac.uk/pub/"
+                        "databases/lrgex/pending/"
+                        f"{lrg_id}.xml"
+                    )
 
         # Ensembl.
         if selected_assembly == "grch37":

VariantValidator/modules/vvMixinConverters.py

@@ -1610,6 +1610,10 @@ def myvm_t_to_g(self, hgvs_c, alt_chr, no_norm_evm, hn, map_dat):
             hgvs_genomic.posedit.edit.alt = hgvs_genomic.posedit.edit.ref
 
         if hgvs_genomic.posedit.edit.type == 'ins' and utilise_gap_code:
+
+            if stored_hgvs_c.posedit.edit.type == "dup":
+                stored_hgvs_c = hgvs_dup_to_delins(stored_hgvs_c)
+
             try:
                 # Can move ins variants (and in doing so break
                 # mid base == original bases assumption)
@@ -1656,7 +1660,7 @@ def myvm_t_to_g(self, hgvs_c, alt_chr, no_norm_evm, hn, map_dat):
 
             except AttributeError as e:
                 if "'Dup' object has no attribute 'alt'" in str(e):
-                    logger.error(
+                    logger.exception(
                         "Code triggered previously in very poor alignment so not "
                         "able to fully test, refer to test_inputs.py tests "
                         "test_alt_gapping_bug: hgvs_genomic: %s, "

VariantValidator/modules/vvMixinInit.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+from functools import lru_cache
 from configparser import ConfigParser
 
 import vvhgvs
@@ -41,6 +42,56 @@ class InitialisationError(Exception):
     pass
 
 
+class CachedSeqFetcher:
+    """
+    LRU cache wrapper for SeqFetcher.
+
+    This class implements the Decorator pattern around SeqFetcher.
+    Only the most frequently repeated sequence lookup is cached using
+    functools.lru_cache(); all other methods and attributes are
+    transparently delegated to the wrapped SeqFetcher.
+
+    The cache exists solely within the current Python process and is
+    discarded when the process exits.
+    """
+
+    def __init__(self, sf):
+        """
+        Wrap an existing SeqFetcher.
+
+        Parameters
+        ----------
+        sf
+            An instantiated SeqFetcher object.
+        """
+        self._sf = sf
+
+    def __getattr__(self, name):
+        """
+        Delegate all uncached methods and attributes to the wrapped
+        SeqFetcher.
+
+        Python only calls __getattr__ when an attribute is not found on
+        CachedSeqFetcher itself. Consequently, only the methods
+        explicitly implemented below are intercepted and cached;
+        everything else behaves exactly as if the original SeqFetcher
+        were being used directly.
+        """
+        return getattr(self._sf, name)
+
+    @lru_cache(maxsize=settings.SEQFETCHER_CACHE_SIZE)
+    def fetch_seq(self, ac, start_i=None, end_i=None):
+        """
+        Retrieve a sequence or sequence slice.
+
+        The cache key is formed from the accession, start coordinate
+        and end coordinate. Repeated requests for the same sequence
+        slice are therefore served directly from memory rather than
+        performing another SeqRepo lookup.
+        """
+        return self._sf.fetch_seq(ac, start_i, end_i)
+
+
 class Mixin:
     """
     Initialise the persistent VariantValidator infrastructure.
@@ -76,6 +127,18 @@ def __init__(self):
             HGVS_SEQREPO_DIR.split('/')[-1]
         """
 
+        # --------------------------------------------------------------
+        # HGVS global configuration
+        # --------------------------------------------------------------
+
+        vvhgvs.global_config.uta.pool_max = 25
+        vvhgvs.global_config.formatting.max_ref_length = 1000000
+
+        if settings.vvHGVS_HDP_CACHE:
+            vvhgvs.global_config.lru_cache.maxsize = settings.vvHGVS_HDP_CACHE_SIZE
+        else:
+            vvhgvs.global_config.lru_cache.maxsize = 200  # Default vvHGVS cache size.
+
         # --------------------------------------------------------------
         # Configuration
         # --------------------------------------------------------------
@@ -226,19 +289,11 @@ def __init__(self):
         self.no_norm_evm = None
 
         # --------------------------------------------------------------
-        # HGVS global configuration
+        # HGVS data providers
         # --------------------------------------------------------------
 
-        vvhgvs.global_config.uta.pool_max = 25
-        vvhgvs.global_config.formatting.max_ref_length = 1000000
-
-        # --------------------------------------------------------------
-        # HGVS data provider
-        # --------------------------------------------------------------
-
-        self.hdp = vvhgvs.dataproviders.uta.connect(
-            pooling=True,
-        )
+        # Create the HGVS data provider.
+        self.hdp = vvhgvs.dataproviders.uta.connect(pooling=True)
 
         self.utaSchema = str(
             self.hdp.data_version()
@@ -283,6 +338,16 @@ def __init__(self):
             self.check_same_thread,
         )
 
+        # Wrap the SeqFetcher with the LRU cache layer.
+        #
+        # CachedSeqFetcher intercepts sequence fetches and caches the returned
+        # sequence slices. All other methods and attributes are transparently
+        # delegated to the original SeqFetcher via __getattr__().
+        #
+        # To disable all SeqFetcher caching, simply comment out the line below.
+        if settings.SEQFETCHER_CACHE:
+            self.sf = CachedSeqFetcher(self.sf)
+
         # --------------------------------------------------------------
         # Persistent alignment-specific normalizers
         # --------------------------------------------------------------