From f686d89204d3b591249e2a38f4f3005c69ad6706 Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Tue, 25 Jan 2022 07:44:23 +0100 Subject: [PATCH] imagej's headless regression no longer causes tests to fail (as these failures only happen in headless mode, see issue #6) also added a logging mechanism to reduce stdout pollution (printing to stdout doesn't play nicely with python's unittest mechanism) --- .gitignore | 1 + Makefile | 2 +- .../Ipr/Lipase/Compute_Globules_Area.py | 3 +- src/ij-plugins/Ipr/Lipase/Detect_Globules.py | 3 +- src/ij-plugins/Ipr/Lipase/Display_Sequence.py | 3 +- src/ij-plugins/Ipr/Lipase/Estimate_White.py | 3 +- .../Ipr/Lipase/Preprocess_Sequence.py | 3 +- src/ij-plugins/Ipr/Lipase/Radial_Profile.py | 3 +- src/lipase/__init__.py | 19 +++++++ src/lipase/catalog.py | 9 +-- src/lipase/imagej/__init__.py | 24 +++++++- src/lipase/imagej/ijimageengine.py | 5 +- src/lipase/lipase.py | 4 +- tests/improc_tests.py | 19 +++---- tests/test0001.py | 55 +++++++++++++------ 15 files changed, 114 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index a8ad2a8..397077a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ *.app *$py.class +__pycache__ \ No newline at end of file diff --git a/Makefile b/Makefile index dfc9309..0c03701 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ TEMP_PATH:=$(shell echo ~/work/lipase/tmp) TESTS_OUTPUT_DATA_PATH:=$(TEMP_PATH) LIB_SRC_FILES=$(shell find ./src/lipase -name "*.py") PLUGINS_SRC_FILES=$(shell find ./src/ij-plugins -name "*.py") -LIPASE_VERSION=1.03 +LIPASE_VERSION=1.04 BUILD_ROOT_PATH:=$(TEMP_PATH)/build PACKAGE_FILE_PATH=$(BUILD_ROOT_PATH)/lipase-$(LIPASE_VERSION).zip diff --git a/src/ij-plugins/Ipr/Lipase/Compute_Globules_Area.py b/src/ij-plugins/Ipr/Lipase/Compute_Globules_Area.py index e7a93bc..ac89071 100644 --- a/src/ij-plugins/Ipr/Lipase/Compute_Globules_Area.py +++ b/src/ij-plugins/Ipr/Lipase/Compute_Globules_Area.py @@ -10,8 +10,9 @@ This plugin estimates the global area of globules # # note: fiji's jython doesn't support encoding keyword +from lipase import logger import sys -print('python version %s' % sys.version) # prints python version +logger.debug('python version %s' % sys.version) from lipase.settings import UserSettings diff --git a/src/ij-plugins/Ipr/Lipase/Detect_Globules.py b/src/ij-plugins/Ipr/Lipase/Detect_Globules.py index ca52179..3d00eee 100644 --- a/src/ij-plugins/Ipr/Lipase/Detect_Globules.py +++ b/src/ij-plugins/Ipr/Lipase/Detect_Globules.py @@ -13,8 +13,9 @@ This imagej plugin detects circular shaped particules in the input image, using # # note: fiji's jython doesn't support encoding keyword +from lipase import logger import sys -print('python version %s' % sys.version) # prints python version +logger.debug('python version %s' % sys.version) from lipase.settings import UserSettings diff --git a/src/ij-plugins/Ipr/Lipase/Display_Sequence.py b/src/ij-plugins/Ipr/Lipase/Display_Sequence.py index 28c427d..d6ec917 100644 --- a/src/ij-plugins/Ipr/Lipase/Display_Sequence.py +++ b/src/ij-plugins/Ipr/Lipase/Display_Sequence.py @@ -14,8 +14,9 @@ # String(label="Please enter your name",description="Name field") name # OUTPUT String greeting +from lipase import logger import sys -print('python version %s' % sys.version) # prints python version +logger.debug('python version %s' % sys.version) from lipase.settings import UserSettings diff --git a/src/ij-plugins/Ipr/Lipase/Estimate_White.py b/src/ij-plugins/Ipr/Lipase/Estimate_White.py index 923e391..125a70c 100644 --- a/src/ij-plugins/Ipr/Lipase/Estimate_White.py +++ b/src/ij-plugins/Ipr/Lipase/Estimate_White.py @@ -18,8 +18,9 @@ # String(label="Please enter your name",description="Name field") name # OUTPUT String greeting +from lipase import logger import sys -print('python version %s' % sys.version) # prints python version +logger.debug('python version %s' % sys.version) from lipase.settings import UserSettings diff --git a/src/ij-plugins/Ipr/Lipase/Preprocess_Sequence.py b/src/ij-plugins/Ipr/Lipase/Preprocess_Sequence.py index 2ea059d..a69a146 100644 --- a/src/ij-plugins/Ipr/Lipase/Preprocess_Sequence.py +++ b/src/ij-plugins/Ipr/Lipase/Preprocess_Sequence.py @@ -28,8 +28,9 @@ # String(label="Please enter your name",description="Name field") name # OUTPUT String greeting +from lipase import logger import sys -print('python version %s' % sys.version) # prints python version +logger.debug('python version %s' % sys.version) from lipase.settings import UserSettings diff --git a/src/ij-plugins/Ipr/Lipase/Radial_Profile.py b/src/ij-plugins/Ipr/Lipase/Radial_Profile.py index 01507a1..ef7fbfb 100644 --- a/src/ij-plugins/Ipr/Lipase/Radial_Profile.py +++ b/src/ij-plugins/Ipr/Lipase/Radial_Profile.py @@ -12,8 +12,9 @@ This imagej plugin computes the radial profile of each pixel of the input image. # # note: fiji's jython doesn't support encoding keyword +from lipase import logger import sys -print('python version %s' % sys.version) # prints python version +logger.debug('python version %s' % sys.version) from lipase.settings import UserSettings diff --git a/src/lipase/__init__.py b/src/lipase/__init__.py index e69de29..114750a 100644 --- a/src/lipase/__init__.py +++ b/src/lipase/__init__.py @@ -0,0 +1,19 @@ +# import sys +import logging +import logging.handlers +import logging +import os + +def create_logger(): + global logger + # logging.basicConfig( stream=sys.stderr ) + handler = logging.FileHandler(os.environ.get("LOGFILE", "/tmp/lipase.log")) + formatter = logging.Formatter(logging.BASIC_FORMAT) + handler.setFormatter(formatter) + logger = logging.getLogger(__name__) + logger.setLevel( logging.DEBUG ) + logger.addHandler(handler) + return logger + +logger = create_logger() + diff --git a/src/lipase/catalog.py b/src/lipase/catalog.py index 6033e8c..a0e29c3 100644 --- a/src/lipase/catalog.py +++ b/src/lipase/catalog.py @@ -2,7 +2,7 @@ import os import json from .imageengine import IHyperStack, IImageEngine, PixelType - +from . import logger class DacMetadata(object): """Represents a display_and_comments.txt metadata file. @@ -40,8 +40,8 @@ class Sequence(object): self.catalog = catalog self.id = sequence_id self.micro_manager_metadata_file_path = micro_manager_metadata_file_path - print(micro_manager_metadata_file_path) - print('reading micro manager metatdata from %s' % micro_manager_metadata_file_path) + logger.debug('reading micro manager metatdata from %s' % micro_manager_metadata_file_path) + # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments) with open(os.path.realpath(micro_manager_metadata_file_path), "r") as mmm_file: self.mmm = json.load(mmm_file, encoding='latin-1') # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments) @@ -49,7 +49,8 @@ class Sequence(object): (pos_dir_path, file_name) = os.path.split(micro_manager_metadata_file_path) # pylint: disable=unused-variable (micro_manager_metadata_file_parent_path, pos_dir_name) = os.path.split(pos_dir_path) # pylint: disable=unused-variable assert pos_dir_name[0:3] == 'Pos', 'unexpected value : %s is expected to be of the form "Pos"' % pos_dir_name - print('micro_manager_metadata_file_parent_path = %s' % micro_manager_metadata_file_parent_path) + logger.debug('micro_manager_metadata_file_parent_path = %s' % micro_manager_metadata_file_parent_path) + dac_file_path = os.path.join(micro_manager_metadata_file_parent_path, 'display_and_comments.txt') (dac_id, file_name) = os.path.split(self.id) self.dac = DacMetadata(dac_id, dac_file_path) diff --git a/src/lipase/imagej/__init__.py b/src/lipase/imagej/__init__.py index 0f1e237..2bcde53 100644 --- a/src/lipase/imagej/__init__.py +++ b/src/lipase/imagej/__init__.py @@ -1,4 +1,5 @@ -from ij import IJ # pylint: disable=import-error +from ij import IJ, ImagePlus # pylint: disable=import-error +import re def open_sequence_as_hyperstack(sequence): @@ -38,3 +39,24 @@ def open_sequence_in_imagej(sequence): hyperstack.setPositionWithoutUpdate(channel_index + 1, 1, 1) IJ.run("Enhance Contrast", "saturated=0.35") return hyperstack + +def ij_version_as_float(ij_version): + ''' + ij_version : something like '1.53f51' + ''' + match = re.match('(?P[0-9]+).(?P[0-9]+)(?P[a-z])(?P[0-9]+)', ij_version) + assert match + n3_as_number = ord(match.group('n3')) - ord('a') + assert n3_as_number >= 0 + assert n3_as_number < 26 + no_letter_version = match.group('n1')+'.'+ match.group('n2') + ('%02d' % (n3_as_number)) + match.group('n4') + return float(no_letter_version) + + +def imagej_has_headless_bug(): + # https://forum.image.sc/t/processing-filter-headless-in-jython/48055/2 + ij_full_version = IJ.getFullVersion() # something like '1.53f51' + this_ij_version = ij_version_as_float(ij_full_version) + return this_ij_version >= ij_version_as_float('1.52q00') and this_ij_version < ij_version_as_float('1.53h55') + + diff --git a/src/lipase/imagej/ijimageengine.py b/src/lipase/imagej/ijimageengine.py index a5b695a..27a237a 100644 --- a/src/lipase/imagej/ijimageengine.py +++ b/src/lipase/imagej/ijimageengine.py @@ -4,6 +4,7 @@ import os.path from ..imageengine import IImage, IHyperStack, IImageEngine, PixelType, IImageProcessingDebugger, NullDebugger from ..maxima_finder import Match +from .. import logger from java.awt import Polygon from ij import IJ, ImagePlus # pylint: disable=import-error from ij.process import FloodFiller # pylint: disable=import-error @@ -480,10 +481,10 @@ class IJImageEngine(IImageEngine): def match_template(self, src_image, template_image): cv_match_template_supported_pixel_types = [PixelType.U8, PixelType.F32] if src_image.get_pixel_type() not in cv_match_template_supported_pixel_types: - print('converting src_image') + logger.debug('converting src_image') src_image = src_image.clone(PixelType.F32) if template_image.get_pixel_type() not in cv_match_template_supported_pixel_types: - print('converting template_image') + logger.debug('converting template_image') template_image = template_image.clone(PixelType.F32) # import org.opencv.imgproc.Imgproc; # diff --git a/src/lipase/lipase.py b/src/lipase/lipase.py index b30c262..7365312 100644 --- a/src/lipase/lipase.py +++ b/src/lipase/lipase.py @@ -17,7 +17,7 @@ from .imageengine import IImageEngine, IImageProcessingDebugger, NullDebugger, S # greeting = "Hello, " + name + "!" from .hdf5.hdf5_data import Group, DataSet, ElementType from .imagej.hdf5serializer import save_hdf5_file - +from . import logger import abc ABC = abc.ABCMeta('ABC', (object,), {}) @@ -210,7 +210,7 @@ class GlobulesAreaEstimator(object): particle_pixel_value = 255.0 num_pixels = is_particle.get_width() * is_particle.get_height() num_particle_pixels = int(measured_mean_value * num_pixels / particle_pixel_value) - print("num_particle_pixels: %d " % num_particle_pixels) + logger.info("num_particle_pixels: %d " % num_particle_pixels) globules_area_ratio = float(num_particle_pixels)/frame_area globules_area[(frame_index, )] = globules_area_ratio frame_indices[(frame_index, )] = frame_index diff --git a/tests/improc_tests.py b/tests/improc_tests.py index 55bb75b..e863063 100644 --- a/tests/improc_tests.py +++ b/tests/improc_tests.py @@ -10,6 +10,7 @@ import sys from lipase.imageengine import IImageEngine, PixelType, Aabb, NullDebugger, FileBasedDebugger, StackImageFeeder from lipase.imagej.ijimageengine import IJImageEngine, IJImage from lipase.circsymdetector import create_circle_image +from lipase import logger class ImProcTester(unittest.TestCase): @@ -18,15 +19,13 @@ class ImProcTester(unittest.TestCase): TESTS_OUTPUT_DATA_PATH = tests_output_data_path # eg '/tmp/lipase/tests-output-data' pylint: disable=undefined-variable def setUp(self): - print("initializing ImProcTester instance") IImageEngine.set_instance(IJImageEngine(debugger=FileBasedDebugger('%s/debug-images' % self.TESTS_OUTPUT_DATA_PATH))) def tearDown(self): - print("uninitializing ImProcTester instance") - + pass def test_create_circle_image(self): - print("executing test_create_circle_image") + logger.info("executing test_create_circle_image") image_width = 21 image_height = 17 circle_image = create_circle_image( @@ -39,8 +38,8 @@ class ImProcTester(unittest.TestCase): measured_mean_value = circle_image.get_mean_value() expected_number_of_circle_pixels = 19 expected_mean_value = float(expected_number_of_circle_pixels) / (image_width * image_height) - print("expected_mean_value: %f" % expected_mean_value) - print("measured_mean_value: %f" % measured_mean_value) + logger.info("expected_mean_value: %f" % expected_mean_value) + logger.info("measured_mean_value: %f" % measured_mean_value) self.assertAlmostEqual(measured_mean_value, expected_mean_value, delta=0.01) def test_stack_mean(self): @@ -61,18 +60,18 @@ class ImProcTester(unittest.TestCase): self.assertAlmostEqual(measured_avg, expected_avg, delta=0.0001) def run_script(): - print("executing run_script") - + logger.debug("executing run_script") + # unittest.main() # this would result in : ImportError: No module named __main__ # solution from : https://discourse.mcneel.com/t/using-unittest-in-rhino-python-not-possible/15364 suite = unittest.TestLoader().loadTestsFromTestCase(ImProcTester) stream = sys.stdout # by default it's sys.stderr, which doesn't appear in imagej's output test_result = unittest.TextTestRunner(stream=stream, verbosity=2).run(suite) - print('test_result : %s' % test_result) + logger.info('test_result : %s' % test_result) # store summary of the result in a file so that the caller of imagej can detect that this python script failed (imagej seems to always return error code 0, regardless the error returned by the python script it executes : even sys.exit(1) doesn't change this) with open('/tmp/test_result.txt', 'w') as f: f.write('%d' % {True: 0, False: 1}[test_result.wasSuccessful()]) - print('end of run_script') + logger.debug('end of run_script') # note : when launched from fiji, __name__ doesn't have the value "__main__", as when launched from python run_script() diff --git a/tests/test0001.py b/tests/test0001.py index 17f3111..abbe0aa 100644 --- a/tests/test0001.py +++ b/tests/test0001.py @@ -20,6 +20,8 @@ from lipase.lipase import Lipase, ImageLogger from lipase.lipase import GlobulesAreaEstimator, EmptyFrameBackgroundEstimator from lipase.circsymdetector import CircularSymmetryDetector, GlobulesDetector from lipase.imagej.hdf5serializer import save_hdf5_file +from lipase.imagej import imagej_has_headless_bug +from lipase import logger def get_trap_area(sequence): if sequence.id == 'res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0': @@ -54,15 +56,16 @@ class TestLipase(unittest.TestCase): # CURRENT_RESULT = None # holds last result object passed to run method def setUp(self): - print("initializing TestLipase instance") IImageEngine.set_instance(IJImageEngine(debugger=FileBasedDebugger('%s/debug-images' % self.TESTS_OUTPUT_DATA_PATH))) self.catalog = ImageCatalog(self.RAW_IMAGES_ROOT_PATH) def tearDown(self): - print("uninitializing TestLipase instance") self.catalog = None def test_estimate_white(self): + if imagej_has_headless_bug(): + logger.warn('skipping test because of headless bug https://github.com/imagej/imagej1/commit/e0e4fc8c3d449faa6ffa360d67e20999691aa362') + return sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos2'] white_estimator = WhiteEstimator(open_size=75, close_size=75, average_size=75) white_estimate = white_estimator.estimate_white([sequence], ['DM300_327-353_fluo']) @@ -73,9 +76,21 @@ class TestLipase(unittest.TestCase): print('end of test_estimate_white') def test_uniform_lighting_correction(self): + if imagej_has_headless_bug(): + logger.warn('skipping test because of headless bug https://github.com/imagej/imagej1/commit/e0e4fc8c3d449faa6ffa360d67e20999691aa362') + return non_uniform_sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0'] uniform_sequence = correct_non_uniform_lighting(non_uniform_sequence, 'DM300_nofilter_vis', white_estimator=WhiteEstimator(open_size=75, close_size=75, average_size=75)) # pylint: disable=unused-variable + + def test_issue6(self): + if imagej_has_headless_bug(): + logger.warn('skipping test because of headless bug https://github.com/imagej/imagej1/commit/e0e4fc8c3d449faa6ffa360d67e20999691aa362') + return + ie = IImageEngine.get_instance() + im = ie.create_image(width=1024, height=1024, pixel_type=PixelType.U16) + IJ.run(im.ij_image, "Mean...", "radius=38") + def test_template_matcher(self): sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0'] stack = sequence.as_hyperstack(['DM300_nofilter_vis'], selected_frames=[0]) @@ -83,7 +98,7 @@ class TestLipase(unittest.TestCase): template_trap_aabb = get_trap_area(sequence) template_trap_image = first_image.get_subimage(template_trap_aabb) for image in [first_image, template_trap_image]: - print(image.get_pixel_type(), image.get_width(), image.get_height()) + logger.info(image.get_pixel_type(), image.get_width(), image.get_height()) # the typical value of peaks is -2.e10 and the value between peaks is below -8.0e10 threshold = -3.0e10 tolerance = 1.0e10 @@ -91,22 +106,26 @@ class TestLipase(unittest.TestCase): template_matcher = TemplateMatcher(maxima_finder) matches = template_matcher.match_template(first_image, template_trap_image) num_traps = len(matches) - print("number of traps found : %d" % num_traps) + logger.info("number of traps found : %d" % num_traps) num_expected_traps = 13 # 13 traps are completely visible in the first image self.assertAlmostEqual(len(matches), num_expected_traps, delta=1.0) def test_traps_detector(self): - sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0'] - traps_mask = get_traps_mask(sequence) - - measured_mean_value = traps_mask.get_mean_value() - expected_traps_coverage = 0.07909 - traps_pixel_value = 255.0 - expected_mean_value = expected_traps_coverage * traps_pixel_value - print("expected_mean_value: %f" % expected_mean_value) - print("measured_mean_value: %f" % measured_mean_value) - self.assertAlmostEqual(measured_mean_value, expected_mean_value, delta=0.01) + if imagej_has_headless_bug(): + logger.warn('skipping test because of headless bug https://github.com/imagej/imagej1/commit/e0e4fc8c3d449faa6ffa360d67e20999691aa362') + return + + sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0'] + traps_mask = get_traps_mask(sequence) + + measured_mean_value = traps_mask.get_mean_value() + expected_traps_coverage = 0.07909 + traps_pixel_value = 255.0 + expected_mean_value = expected_traps_coverage * traps_pixel_value + print("expected_mean_value: %f" % expected_mean_value) + print("measured_mean_value: %f" % measured_mean_value) + self.assertAlmostEqual(measured_mean_value, expected_mean_value, delta=0.01) def test_visible_traps_sequence_processing(self): traps_sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0'] @@ -129,6 +148,10 @@ class TestLipase(unittest.TestCase): radial_profiles, angular_stddev_profiles = detector.compute_radial_profiles(src_image) # pylint: disable=unused-variable def test_globules_detector(self): + if imagej_has_headless_bug(): + logger.warn('skipping test because of headless bug https://github.com/imagej/imagej1/commit/e0e4fc8c3d449faa6ffa360d67e20999691aa362') + return + traps_sequence = self.catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos0'] visible_traps_sequence = traps_sequence.as_hyperstack(['DM300_nofilter_vis']) traps_mask = get_traps_mask(traps_sequence) @@ -160,11 +183,11 @@ def run_script(): suite = unittest.TestLoader().loadTestsFromTestCase(TestLipase) stream = sys.stdout # by default it's sys.stderr, which doesn't appear in imagej's output test_result = unittest.TextTestRunner(stream=stream, verbosity=2).run(suite) - print('test_result : %s' % test_result) + logger.info('test_result : %s' % test_result) # store summary of the result in a file so that the caller of imagej can detect that this python script failed (imagej seems to always return error code 0, regardless the error returned by the python script it executes : even sys.exit(1) doesn't change this) with open('/tmp/test_result.txt', 'w') as f: f.write('%d' % {True: 0, False: 1}[test_result.wasSuccessful()]) - print('end of run_script') + logger.debug('end of run_script') # note : when launched from fiji, __name__ doesn't have the value "__main__", as when launched from python run_script()