started to translate matlab telemosToolbox's preprocessing functions into jython so that they can benefit from fiji framework

- started to write estimate_white, with its unit test
- refactor: split lipase.py into modules
- cleanup : removed unused files and unused jenkins steps
- improved jenkinsfile

Bug 2623 - Faire un traitement automatique pour les images du projet lipase

.gitignore

@@ -28,3 +28,4 @@
 *.out
 *.app
 
+*$py.class

Jenkinsfile

@@ -4,49 +4,33 @@ pipeline {
         stage('Initial setup...') {
             steps {
                 echo 'Initial setup...'
+                sh './scripts/install_fiji.bash'
                 // echo 'Create or update the virtual Python environment'
                 // sh '/bin/bash ./src/CI/CI.bash -i ci_venv'
             }
         }
-        stage('Building the package...') {
-            steps {
-                sh '/bin/bash -c make'
-                // sh '/bin/bash ./src/CI/CI.bash -p ci_venv'
-            }
-        }
         stage('Testing the package...') {
             steps {
-                sh './hello'
+                sh './tests/test0001.bash'
-                // sh 'cp ~/.bashrc ~/.bashrc.bak'
-                // sh './install_resources/packages/$(cat VERSION).setup --accept -- -y'
-                // sh '$HOME/.local/bin/msspec -f $HOME/.local/share/$(cat VERSION)'
-                // sh '$HOME/.local/bin/msspec -t'
-                // sh 'yes 1|$HOME/.local/bin/msspec -u'
-                // sh 'mv ~/.bashrc.bak ~/.bashrc'
-            }
-        }
-        stage('Testing image processing...') {
-            steps {
-                sh './test1.bash'
-            }
-        }
-        stage('Building HTML documentation...') {
-            steps {
-                echo 'Building HTML documentation...'
-                // sh '/bin/bash ./src/CI/CI.bash -d ci_venv'
-            }
-        }
-        stage('Releasing package...') {
-            steps {
-                echo 'Releasing package...'
-                // sh 'rm -rf $HOME/www/*'
-                // sh 'cp -a ./src/doc/build/html/* $HOME/www/'
             }
         }
+        // stage('Building HTML documentation...') {
+        //     steps {
+        //         echo 'Building HTML documentation...'
+        //         // sh '/bin/bash ./src/CI/CI.bash -d ci_venv'
+        //     }
+        // }
+        // stage('Releasing package...') {
+        //     steps {
+        //         echo 'Releasing package...'
+        //         // sh 'rm -rf $HOME/www/*'
+        //         // sh 'cp -a ./src/doc/build/html/* $HOME/www/'
+        //     }
+        // }
         stage('Cleaning up...') {
             steps {
                 echo 'Cleaning artifacts...'
-                sh '/bin/bash -c "make clean"'
+                sh './scripts/uninstall_fiji.bash'
                 // sh 'rm -rf ./install_resources'
                 // sh 'cd ./src/doc && make clean'
             }

catalog.py

@@ -2,16 +2,44 @@
 import json
 
 
+class DacMetadata(object):
+    """Represents a display_and_comments.txt metadata file"""
+    def __init__(self, dac_id, dac_file_path):
+        """Contructor.
+
+        :param str dac_id: eg "res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1"
+        :param str dac_file_path: eg "/Users/graffy/ownCloud/ipr/lipase/raw-images/res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/display_and_comments.txt"
+        """
+        self.dac_id = dac_id
+        self.dac_file_path = dac_file_path
+        with open(dac_file_path, "r") as dac_file:
+            self.dac = json.load(dac_file, encoding='latin-1')  # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments)
+
+    def get_channel_name(self, channel_index):
+        channels = self.dac['Channels']
+        return channels[channel_index]["Name"]
+
+
 class Sequence(object):
     def __init__(self, catalog, sequence_id, micro_manager_metadata_file_path):
         self.catalog = catalog
         self.sequence_id = sequence_id
         self.micro_manager_metadata_file_path = micro_manager_metadata_file_path
         print(micro_manager_metadata_file_path)
-
+        print('reading micro manager metatdata from %s' % micro_manager_metadata_file_path)
+        # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments)
         with open(micro_manager_metadata_file_path, "r") as mmm_file:
             self.mmm = json.load(mmm_file, encoding='latin-1')  # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments)
 
+        micro_manager_metadata_file_parent_path = '/'.join(micro_manager_metadata_file_path.split('/')[0:-2])
+        print('micro_manager_metadata_file_parent_path = %s' % micro_manager_metadata_file_parent_path)
+        dac_file_path = micro_manager_metadata_file_parent_path + '/display_and_comments.txt'
+        dac_id = '/'.join(self.sequence_id.split('/')[0:-1])
+        self.dac = DacMetadata(dac_id, dac_file_path)
+
+        for channel_index in range(self.num_channels):
+            assert self.get_channel_name(channel_index) == self.dac.get_channel_name(channel_index), 'mismatched channel names for channel index %d : %s in %s, %s in %s' % (channel_index, self.get_channel_name(channel_index), self.micro_manager_metadata_file_path, self.dac.get_channel_name(channel_index), self.dac.dac_file_path)
+
     @property
     def num_frames(self):
         summary = self.mmm['Summary']
@@ -65,6 +93,10 @@ class Sequence(object):
         channel_index = summary['ChNames'].index(channel_id)
         return channel_index
 
+    def get_channel_name(self, channel_index):
+        summary = self.mmm['Summary']
+        return summary['ChNames'][channel_index]
+
     def get_black(self):
         ''' returns the black sequence related to the the sequence self
 

hello.cpp

@@ -1,8 +0,0 @@
-#include <iostream>
-
-using namespace std;
-
-int main(int argc, char * argv[])
-{
-	cout << "hello, world!" << endl;
-}

lipase.py

@@ -1,11 +1,5 @@
-# note: fiji's jython doesn't support encoding keyword
-# String(label="Please enter your name",description="Name field") name
-# OUTPUT String greeting
-import sys
-print(sys.version)  # prints python version
-
-sys.path.append('/Users/graffy/ownCloud/ipr/lipase/lipase.git')  # necessary if run from fiji as script otherwise jython fails to find lipase's modules such as catalog
 
+# encoding: utf8
 # A Jython script with parameters.
 # It is the duty of the scripting framework to harvest
 # the 'name' parameter from the user, and then display
@@ -18,6 +12,7 @@ from ij.plugin import ImageCalculator
 from ij.plugin import ZProjector
 from ij import WindowManager
 from catalog import Sequence, ImageCatalog
+import preprocessing
 # greeting = "Hello, " + name + "!"
 
 
@@ -183,6 +178,9 @@ class Lipase(object):
         background_image.show()
         return background_image
 
+    def estimate_white(self, sequence, channel_id):
+        return preprocessing.estimate_white([sequence], [channel_id], dark=None)
+
     def process_sequence(self, sequence_id):
         '''
         :param str sequence_id:
@@ -210,6 +208,7 @@ def test_find_white():
     depth_index = lipase.find_depth_index(src_image, white_seq)
 
 
+
 def run_script():
     test_find_white()
 

preprocessing.py

@@ -0,0 +1,271 @@
+"""preprocessing of synchrotron images based on telemosToolbox."""
+
+from ij import IJ
+from ij.plugin import ImageCalculator
+from catalog import ImageCatalog, Sequence
+
+
+def compute_max(images_file_path):
+    """Computes for each pixel position the maximum at this position in all input images.
+
+    :param list(str) images_file_path:
+    :rtype ImagePlus:
+    """
+    assert len(images_file_path) > 1
+    max_image = IJ.openImage(images_file_path[0])
+    print('max_image', max_image)
+
+    for image_file_path in images_file_path[2:-1]:
+        other_image = IJ.openImage(image_file_path)
+        print('other_image', other_image)
+        ic = ImageCalculator()
+        ic.run("max", max_image, other_image)
+    print('max_image', max_image)
+    return max_image
+
+
+def replace_outer_frame(image, band_width):
+    """Overwrites the outer band of the image by duplicating the value of the pixels that touch this outer band
+
+    :param ImagePlus image:
+    :param int band_width: width of the outer band, in pixels
+    :rtype ImagePlus:
+
+    adaptation for the following code from matlab telemosToolbx's estimatedwhiteFluoImageTelemos function
+        outer = white_estimate;
+        white_estimate = white_estimate(4:(end-3),4:(end-3));
+        outer(1:3,4:(end-3))=repmat(white_estimate(1,:),3,1);  # top
+        outer((end-2):end,4:(end-3))=repmat(white_estimate(end,:),3,1); # bottom
+        outer(:,1:3)=repmat(outer(:,4),1,3); # left
+        outer(:,(end-2):end)=repmat(outer(:,end-3),1,3); # right
+        white_estimate = outer;
+        clear outer;
+    """
+    raise NotImplementedError()
+    return image
+
+
+
+def estimate_white(sequences, channel_ids, dark=None):
+    """Estimation of the white fluorescence image shape of synchrotron light from experimental images of Telemos microscope.
+
+    :param list(Sequence) sequences:
+    :param list(str) channel_ids:
+    :param ImagePlus or None dark:
+    :rtype ImagePlus:
+    Code adapted from matlab telemosToolbx's estimatedwhiteFluoImageTelemos function
+
+    % this function is specific of Telemos images (DISCO line SOLEIL)
+        % acquired using micromanager software linked to imageJ
+
+    %% input
+
+    %  nothing : interactive function
+
+    %% output
+
+    %  nothing : the final reference fluorescence image is saved on disk
+
+    %% principe
+    %  NB: MICROMANAGER save images in a structured file folder architecture :
+    %
+    % root folder : name given by the user
+    %
+    %   subfolders roi(n)_tile1 : n folders for each selected roi
+    %           or pos(n) : n folders for each selected position
+    %
+    %   display_and_comments.txt : file describing the channels acquired
+    %
+    % in each subfolder roi(n)_tile1 or Pos(n) :
+    %       images files with name img_00000000(n)_DM300_327-353_00(p).tif
+    %               n = number of time
+    %               p = z focal plane
+    %               DM300_327-353 = name of the channel
+    %       metadata.txt : files describing all metadata associated with the
+    %       acquisition : x, y, z position, camera settings .... etc.
+    %
+    % (Nb: depending on the date of acquisition, an extension fluo is found or
+    % not in the name of fluorescence image. This extension is given in
+    % contrast to the visible image.)
+    %
+    % IMAGES : fluorescence images are considered
+    %
+    %
+    % IMPORTANT:
+    %       it is expected that in the folders at least several images cover the whole field of view
+    %       illuminated by the synchrtron light
+    %       IF IT IS NOT THE CASE, IT WON'T WORK
+    %
+    % BASIC: the estimated white image is the MAX of the selected images
+    %       when interactive mode is selected, only first time and first z are
+    %       proposed
+    %       for automatic computing, all images are taken into account
+    %
+    % WHAT IS DONE:
+    %        0 - the user chosse the channels that are considered  in the estimation of the
+    % white fluorescence image
+    %
+    %       1 - spurious pixels (three lines and columns around the image) are not taken
+    % in filtering and replaced by line and column number 4 and end-3
+    %
+    %       2 - Filtering
+    % the estimated dark image is filtered :
+    %        opening to remove  white regions brighter than the low frequency signal that should correspond to  synchrotron light shape (should be small)
+    %        closing to remove black regions that remains (should be small to avoid synchrotron light shape deformation)
+    %        average filtering
+
+    % the image computed should be used  to
+    %   - to find the best white z plane from the reference white stack
+    %   acquired for ex. using the so called "Matthieu lame"%
+    %   - correct images for intensities
+
+    %% use
+    %       estimatedwhiteFluoImageTelemos
+
+    %% Comments
+    %   adapted from proposals 20161050 and 20171187
+
+
+    %% Author
+    % MF Devaux
+    % INRA BIA
+    % PVPP
+
+    %% date
+    % 5 octobre 2017:
+    % 23 mars 2018
+    % 3 septembre 2018 : comments and general case
+    % 16 avril 2019: name of function and default filtering values
+    % 29 avril 2019 : new comments and spurious pixels
+    % 27 mai 2019 : offset dark
+    %     4 juin 2019 : replace exist by isfolder or isfile
+    % 14 juin : close figure at the end
+    """
+    images_file_path = []
+    for sequence in sequences:
+        for channel_id in channel_ids:
+            channel_index = sequence.get_channel_index(channel_id)
+            for frame_index in range(sequence.num_frames):
+                for z_index in range(sequence.num_slices):
+                    images_file_path.append(sequence.get_image_file_path(channel_index, frame_index, z_index))
+
+    white_estimate = compute_max(images_file_path)
+    
+    # modifiy spurious pixels on the side of the images
+    try:
+        replace_outer_frame(white_estimate, 3)
+    except NotImplementedError as error:
+        print('warning: replace_outer_frame is not implemented yet')
+
+    
+    return white_estimate
+
+
+def find_white_reference_image(white_estimate, white_z_stack):
+    """Find the white reference Image among a z stack of reference image the most correlated to the white image estimated from a serie of acquisition.
+
+    :param Image white_estimate:
+    :param Sequence white_z_stack:
+    % this function is specific of Telemos images (DISCO line SOLEIL)
+        % acquired using micromanager software linked to imageJ
+
+    %% input
+    %  nothing : interactive function
+
+    %% output
+    %    zmax : focal plane
+
+    %% principe
+    % images were acquired for a given roi which position is found in
+    % metadata.txt file
+    % white and dark  images were recorded for the full field of view of the DISCO
+    % TELEMOS camera
+    %
+    % correlation between a estimated  white fluorescence image estalished from
+    % actual acquisitions of a given sample and all z plane acquired for the reference TELEMOS white
+    % image (Matthieu slide or any fluorescent homogeneous image) :
+
+    % The  estimated  white fluorescence image  is generallly obtained bt using function whiteFluoImageTelemosestimation
+    % This is not compulsary as any homogenous sample image hat can roughly show the shape of illumination can be used to find
+    % the white reference image
+    %
+    %
+    % the z plane for which the maximum correlation is observed between estimated white and reference white images is retained.
+    % the white image is then offsetted (its corresponding Dark is subtracted) and copied in the subfolder <WhiteReference> of the sample
+    % rootfolder to show that it has been specifically selected for the considered experiment
+    %The matlab corrcoeff function is used
+    %
+    % correlation coefficients are saved in a file called
+    % 'corr.txt' in the subfolder 'WhiteReference'
+    %
+    %
+    % expected input folder hierarchy:
+    %
+    %      >sampleFolder
+    %           > PosFolders or RoiFolders
+    %           > WhiteEstimate
+    %       >darkFolder
+    %       >darkFolder.smooth
+    %       >whiteFolder
+    %           > darkFolderforWhite
+    %           > darkFolderforWhite.smooth
+    %       >whiteFolder.smooth
+    %
+    %
+    % expected output folder hierarchy:
+    %
+    %      >sampleFolder
+    %           > PosFolders or RoiFolders
+    %           > WhiteEstimate
+    %           > WhiteReference
+    %                   > white after offset
+    %       >darkFolder
+    %       >darkFolder.smooth
+    %       >whiteFolder
+    %           > darkFolderforWhite
+    %           > darkFolderforWhite.smooth
+    %       >whiteFolder.smooth
+
+
+
+
+
+    %% use
+    % [zmax]=findWhiteReferenceImageTelemos
+
+    %% Comments
+    %   written for  proposal 20161050
+    %   adapted for proposal 20170043
+
+
+    %% Author
+    % MF Devaux
+    % INRA BIA
+    % PVPP
+
+    %% date
+    % 5 octobre 2017:
+    % 15 decembre 2017 : adapted to take into account the roi known from
+    % metadata
+    % 27 fevrier 2018 : comments details
+    % 4 septembre 2018: comments and check and naming of folders
+    % 12 mars 2019 : save white reference with the same size as white estimate
+    % 16 avril 2019 : include diagonals to check the relevance of white
+    % reference
+    % 20 mai 2019 : track folder
+    % 27 mai 2019 : offset dark
+    %     4 juin 2019 : replace exist by isfolder or isfile
+    """
+    raise NotImplementedError()
+
+
+def test_preprocessing():
+    """Test preprocessing."""
+    catalog = ImageCatalog('/Users/graffy/ownCloud/ipr/lipase/raw-images')
+    sequence = catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos2']
+    white_estimate = estimate_white([sequence], ['DM300_327-353_fluo'])
+    find_white_reference_image(white_estimate, sequence.get_white())
+
+
+if __name__ == '__main__':
+    test_preprocessing()

scripts/install_fiji.bash

@@ -2,5 +2,3 @@
 wget 'https://downloads.imagej.net/fiji/latest/fiji-linux64.zip'
 unzip './fiji-linux64.zip'
 rm './fiji-linux64.zip'
-./Fiji.app/ImageJ-linux64 --ij2 --headless --run './lipase.py'
-

scripts/uninstall_fiji.bash

@@ -0,0 +1,2 @@
+#!/bin/bash
+rm -R './Fiji.app'

tests/test0001.bash

@@ -0,0 +1,3 @@
+./Fiji.app/ImageJ-linux64 --ij2 --headless --run './tests/test0001.py' "raw_images_root_path='/opt/ipr/cluster/work.global/graffy/jenkins-store/lipase/raw-images'"
+# on macosx : /Applications/Fiji.app/Contents/MacOS/ImageJ-macosx --ij2 --headless --run './test0001.py'
+

tests/test0001.py

@@ -0,0 +1,29 @@
+"""This script is supposed to be launched from fiji's jython interpreter
+"""
+# # note: fiji's jython doesn't support encoding keyword
+
+#https://imagej.net/Scripting_Headless
+#@String raw_images_root_path
+
+# String(label="Please enter your name",description="Name field") name
+# OUTPUT String greeting
+import sys
+print('python version %s' % sys.version)  # prints python version
+
+sys.path.append('/Users/graffy/ownCloud/ipr/lipase/lipase.git')  # necessary if run from fiji as script otherwise jython fails to find lipase's modules such as catalog
+
+from ij import IJ
+from lipase import Lipase, ImageLogger
+from catalog import ImageCatalog
+
+def run_script():
+    catalog = ImageCatalog(raw_images_root_path)  # eg '/Users/graffy/ownCloud/ipr/lipase/raw-images'
+    lipase = Lipase(catalog, debugger=ImageLogger())
+    sequence = catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos2']
+    white_estimate = lipase.estimate_white(sequence, 'DM300_327-353_fluo')
+    print(white_estimate)
+    IJ.saveAsTiff(white_estimate, './white_estimate.tiff')
+    print('end')
+
+# note : when launched from fiji, __name__ doesn't have the value "__main__", as when launched from python
+run_script()