From 9ea8fcbe108a84f2729d727268385b6dc0c37fcc Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Fri, 18 Mar 2022 18:22:10 +1100 Subject: [PATCH 1/8] Replace references to log4j with logback --- .gitignore | 2 +- doc/source/bioformats.rst | 16 ++++++------- pims/bioformats.py | 48 +++++++++++++++++++-------------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index db8ecc77..3e6af317 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ dist/* pims.egg-info/* PIMS.egg-info/* pims/version.py -pims/loci_tools.jar +pims/bioformats_package.jar examples/.ipynb_checkpoints pims/tests/data/bulk-water.mov.pims_buffer pims/tests/data/bulk-water.mov.pims_meta diff --git a/doc/source/bioformats.rst b/doc/source/bioformats.rst index 1369db7c..447d18d9 100644 --- a/doc/source/bioformats.rst +++ b/doc/source/bioformats.rst @@ -33,7 +33,7 @@ Or, for windows users, download the binary from `Christoph Gohlke's website `_. On first use of pims.Bioformats(filename), the required java library -:file:`loci_tools.jar` will be automatically downloaded from +:file:`bioformats_package.jar` will be automatically downloaded from `openmicroscopy.org `__. Special functions @@ -95,17 +95,17 @@ To update the version of bioformats you are using in pims: Now you should be able to use pims with the updated bioformats version. -Note: This pims command downloads a bioformats file named `loci_tools.jar` +Note: This pims command downloads a bioformats file named `bioformats_package.jar` to your computer. There are a few possible locations where it might be stored. The precedence order is (highest priority first): 1. pims package location -2. PROGRAMDATA/pims/loci_tools.jar -3. LOCALAPPDATA/pims/loci_tools.jar -4. APPDATA/pims/loci_tools.jar -5. /etc/loci_tools.jar -6. ~/.config/pims/loci_tools.jar +2. PROGRAMDATA/pims/bioformats_package.jar +3. LOCALAPPDATA/pims/bioformats_package.jar +4. APPDATA/pims/bioformats_package.jar +5. /etc/bioformats_package.jar +6. ~/.config/pims/bioformats_package.jar If you encounter problems updating to the latest version of bioformats, -you may wish to manually remove `loci_tools.jar` from each of the six locations +you may wish to manually remove `bioformats_package.jar` from each of the six locations and re-run the `pims.bioformats.download_jar` command again. diff --git a/pims/bioformats.py b/pims/bioformats.py index 60e1805b..43ea5b3a 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -17,15 +17,15 @@ def available(): def _gen_jar_locations(): """ - Generator that yields optional locations of loci_tools.jar. + Generator that yields optional locations of bioformats_package.jar. The precedence order is (highest priority first): 1. pims package location - 2. PROGRAMDATA/pims/loci_tools.jar - 3. LOCALAPPDATA/pims/loci_tools.jar - 4. APPDATA/pims/loci_tools.jar - 5. /etc/loci_tools.jar - 6. ~/.config/pims/loci_tools.jar + 2. PROGRAMDATA/pims/bioformats_package.jar + 3. LOCALAPPDATA/pims/bioformats_package.jar + 4. APPDATA/pims/bioformats_package.jar + 5. /etc/bioformats_package.jar + 6. ~/.config/pims/bioformats_package.jar """ yield os.path.dirname(__file__) if 'PROGRAMDATA' in os.environ: @@ -40,14 +40,14 @@ def _gen_jar_locations(): def _find_jar(): """ - Finds the location of loci_tools.jar, if necessary download it to a + Finds the location of bioformats_package.jar, if necessary download it to a writeable location. """ for loc in _gen_jar_locations(): - if os.path.isfile(os.path.join(loc, 'loci_tools.jar')): - return os.path.join(loc, 'loci_tools.jar') + if os.path.isfile(os.path.join(loc, 'bioformats_package.jar')): + return os.path.join(loc, 'bioformats_package.jar') - warn('loci_tools.jar not found, downloading') + warn('bioformats_package.jar not found, downloading') return download_jar() @@ -68,23 +68,23 @@ def download_jar(version='6'): else: raise IOError('No writeable location found. In order to use the ' 'Bioformats reader, please download ' - 'loci_tools.jar to the pims program folder or one of ' + 'bioformats_package.jar to the pims program folder or one of ' 'the locations provided by _gen_jar_locations().') - url = ('http://downloads.openmicroscopy.org/bio-formats/' + version + - '/artifacts/loci_tools.jar') + url = ('https://downloads.openmicroscopy.org/bio-formats/' + version + + '/artifacts/bioformats_package.jar') - path = os.path.join(loc, 'loci_tools.jar') - loci_tools = urlopen(url).read() + path = os.path.join(loc, 'bioformats_package.jar') + bioformats_package = urlopen(url).read() sha1_checksum = urlopen(url + '.sha1').read().split(b' ')[0].decode() - downloaded = hashlib.sha1(loci_tools).hexdigest() + downloaded = hashlib.sha1(bioformats_package).hexdigest() if downloaded != sha1_checksum: - raise IOError("Downloaded loci_tools.jar has invalid checksum. " + raise IOError("Downloaded bioformats_package.jar has invalid checksum. " "Please try again.") with open(path, 'wb') as output: - output.write(loci_tools) + output.write(bioformats_package) return path @@ -332,7 +332,7 @@ def __init__(self, filename, meta=True, java_memory='512m', if not jpype.isJVMStarted(): from distutils.version import LooseVersion - loci_path = _find_jar() + bioformats_package_path = _find_jar() # If we can turn off string auto-conversion, do so, # since this is the recommended practice. if LooseVersion(jpype.__version__) >= LooseVersion('0.7.0'): @@ -340,12 +340,12 @@ def __init__(self, filename, meta=True, java_memory='512m', else: startJVM_kwargs = {} # convertStrings kwarg not supported for earlier jpype versions jpype.startJVM(jpype.getDefaultJVMPath(), '-ea', - '-Djava.class.path=' + loci_path, + '-Djava.class.path=' + bioformats_package_path, '-Xmx' + java_memory, **startJVM_kwargs) - log4j = jpype.JPackage('org.apache.log4j') - log4j.BasicConfigurator.configure() - log4j_logger = log4j.Logger.getRootLogger() - log4j_logger.setLevel(log4j.Level.ERROR) + logback = jpype.JPackage('org.apache.logback') + logback.BasicConfigurator.configure() + logback_logger = logback.Logger.getRootLogger() + logback_logger.setLevel(logback.Level.ERROR) if not jpype.isThreadAttachedToJVM(): jpype.attachThreadToJVM() From 1598c8aa4e9289ced0d0e73fd1b6c8ff26f0dc7c Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Fri, 18 Mar 2022 18:23:02 +1100 Subject: [PATCH 2/8] Pull latest version of bioformats_package.jar file by default --- pims/bioformats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pims/bioformats.py b/pims/bioformats.py index 43ea5b3a..1190cb3d 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -51,7 +51,7 @@ def _find_jar(): return download_jar() -def download_jar(version='6'): +def download_jar(version='latest'): """ Downloads the bioformats distribution of given version. """ from urllib.request import urlopen import hashlib From 1e483987154ca879c20c1cc619319bd482e2bf1d Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Fri, 18 Mar 2022 18:25:55 +1100 Subject: [PATCH 3/8] Update bioformats documentation to avoid ImportError --- doc/source/bioformats.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/bioformats.rst b/doc/source/bioformats.rst index 447d18d9..8e25cb73 100644 --- a/doc/source/bioformats.rst +++ b/doc/source/bioformats.rst @@ -44,7 +44,7 @@ switches between them: .. code-block:: python # open a multi-experiment file and read the first experiment - reader = pims.BioformatsReader('path/to/file', series=0) + reader = pims.bioformats.BioformatsReader('path/to/file', series=0) # switch to the third experiment reader.series = 2 @@ -53,7 +53,7 @@ open a file with for instance 1 GB of java memory: .. code-block:: python - reader = BioformatsReader('path/to/file', java_memory='1024m') + reader = pims.bioformats.BioformatsReader('path/to/file', java_memory='1024m') Metadata -------- From 493295f9680ac798524b770e7c9c62ea55b75ba6 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Fri, 18 Mar 2022 18:27:12 +1100 Subject: [PATCH 4/8] Temporarily remove non-functional java logger code --- pims/bioformats.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pims/bioformats.py b/pims/bioformats.py index 1190cb3d..d7f1f475 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -342,10 +342,6 @@ def __init__(self, filename, meta=True, java_memory='512m', jpype.startJVM(jpype.getDefaultJVMPath(), '-ea', '-Djava.class.path=' + bioformats_package_path, '-Xmx' + java_memory, **startJVM_kwargs) - logback = jpype.JPackage('org.apache.logback') - logback.BasicConfigurator.configure() - logback_logger = logback.Logger.getRootLogger() - logback_logger.setLevel(logback.Level.ERROR) if not jpype.isThreadAttachedToJVM(): jpype.attachThreadToJVM() From fb0b988eb6684f917fa7f1c6898fa68167eee787 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 21 Mar 2022 14:16:48 +1100 Subject: [PATCH 5/8] Java logback configuration --- pims/bioformats.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pims/bioformats.py b/pims/bioformats.py index d7f1f475..0faae13f 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -1,3 +1,4 @@ +from turtle import pd import numpy as np from pims.base_frames import FramesSequence, FramesSequenceND @@ -342,6 +343,10 @@ def __init__(self, filename, meta=True, java_memory='512m', jpype.startJVM(jpype.getDefaultJVMPath(), '-ea', '-Djava.class.path=' + bioformats_package_path, '-Xmx' + java_memory, **startJVM_kwargs) + logback = jpype.JPackage('ch.qos.logback.classic') + logger_context = logback.LoggerContext() + logback.BasicConfigurator().setContext(logger_context) + logback.BasicConfigurator().configure(logger_context) if not jpype.isThreadAttachedToJVM(): jpype.attachThreadToJVM() From 0f4c69e6032f2852b318a3538ae5a2c296652b37 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:27:12 +1000 Subject: [PATCH 6/8] Set root log level with loci DebugTools https://docs.openmicroscopy.org/bio-formats/6.9.1/developers/logging.html Approach suggested by Josh Moore --- pims/bioformats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pims/bioformats.py b/pims/bioformats.py index 0faae13f..0d82e627 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -352,6 +352,7 @@ def __init__(self, filename, meta=True, java_memory='512m', jpype.attachThreadToJVM() loci = jpype.JPackage('loci') + loci.common.DebugTools.enableLogging('ERROR') # Initialize reader and metadata self.filename = str(filename) From 662aa706a9964ea01de1d2af508c55468f334112 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Thu, 12 May 2022 14:36:00 +1000 Subject: [PATCH 7/8] Replace deprecated distutils LooseVersion with packaging Version --- pims/bioformats.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pims/bioformats.py b/pims/bioformats.py index 0d82e627..2bb81042 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -331,12 +331,14 @@ def __init__(self, filename, meta=True, java_memory='512m', # Start java VM and initialize logger (globally) if not jpype.isJVMStarted(): - from distutils.version import LooseVersion + from packaging.specifiers import SpecifierSet + from packaging.version import Version bioformats_package_path = _find_jar() # If we can turn off string auto-conversion, do so, # since this is the recommended practice. - if LooseVersion(jpype.__version__) >= LooseVersion('0.7.0'): + version_spec = SpecifierSet(">=0.7.0") + if Version(jpype.__version__) in version_spec: startJVM_kwargs = {'convertStrings': False} else: startJVM_kwargs = {} # convertStrings kwarg not supported for earlier jpype versions @@ -348,8 +350,8 @@ def __init__(self, filename, meta=True, java_memory='512m', logback.BasicConfigurator().setContext(logger_context) logback.BasicConfigurator().configure(logger_context) - if not jpype.isThreadAttachedToJVM(): - jpype.attachThreadToJVM() + if not jpype.java.lang.Thread.isAttached(): + jpype.java.lang.Thread.attach() loci = jpype.JPackage('loci') loci.common.DebugTools.enableLogging('ERROR') @@ -409,8 +411,8 @@ def __init__(self, filename, meta=True, java_memory='512m', if isinstance(Jarr[:], np.ndarray): read_mode = 'jpype' else: - warn('Due to an issue with JPype 0.6.0, reading is slower. ' - 'Please consider upgrading JPype to 0.6.1 or later.') + # warn('Due to an issue with JPype 0.6.0, reading is slower. ' + # 'Please consider upgrading JPype to 0.6.1 or later.') try: im = self._jbytearr_stringbuffer(Jarr) im.reshape(self._sizeRGB, self._sizeX, self._sizeY) From 9f0cccc2e65216395097b8ca2ee3451e14367f78 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Thu, 12 May 2022 16:10:12 +1000 Subject: [PATCH 8/8] Update unittest syntaxto more modern version --- pims/tests/test_bioformats.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pims/tests/test_bioformats.py b/pims/tests/test_bioformats.py index abc1b3d8..34984638 100644 --- a/pims/tests/test_bioformats.py +++ b/pims/tests/test_bioformats.py @@ -504,5 +504,4 @@ def test_metadata_tags(self): assert 'PixelsPhysicalSizeX' in fields if __name__ == '__main__': - unittest.runmodule(argv=[__file__, '-vvs'], - exit=False) + unittest.main(verbosity=2)