graffy
/
lipase
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

started to translate matlab telemosToolbox's preprocessing functions into jython so that they can benefit from fiji framework 

- started to write estimate_white, with its unit test
- refactor: split lipase.py into modules
- cleanup : removed unused files and unused jenkins steps
- improved jenkinsfile

Bug 2623 - Faire un traitement automatique pour les images du projet lipase
This commit is contained in:
Guillaume Raffy 2019-07-17 16:56:05 +02:00
parent 76607c19fc
commit 04e46b27dd
10 changed files with 361 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -28,3 +28,4 @@
*.out
*.app
*$py.class

48
Jenkinsfile vendored
View File

@ -4,49 +4,33 @@ pipeline {
stage('Initial setup...') {
steps {
echo 'Initial setup...'
sh './scripts/install_fiji.bash'
// echo 'Create or update the virtual Python environment'
// sh '/bin/bash ./src/CI/CI.bash -i ci_venv'
}
}
stage('Building the package...') {
steps {
sh '/bin/bash -c make'
// sh '/bin/bash ./src/CI/CI.bash -p ci_venv'
}
}
stage('Testing the package...') {
steps {
sh './hello'
// sh 'cp ~/.bashrc ~/.bashrc.bak'
// sh './install_resources/packages/$(cat VERSION).setup --accept -- -y'
// sh '$HOME/.local/bin/msspec -f $HOME/.local/share/$(cat VERSION)'
// sh '$HOME/.local/bin/msspec -t'
// sh 'yes 1|$HOME/.local/bin/msspec -u'
// sh 'mv ~/.bashrc.bak ~/.bashrc'
}
}
stage('Testing image processing...') {
steps {
sh './test1.bash'
}
}
stage('Building HTML documentation...') {
steps {
echo 'Building HTML documentation...'
// sh '/bin/bash ./src/CI/CI.bash -d ci_venv'
}
}
stage('Releasing package...') {
steps {
echo 'Releasing package...'
// sh 'rm -rf $HOME/www/*'
// sh 'cp -a ./src/doc/build/html/* $HOME/www/'
sh './tests/test0001.bash'
}
}
// stage('Building HTML documentation...') {
// steps {
// echo 'Building HTML documentation...'
// // sh '/bin/bash ./src/CI/CI.bash -d ci_venv'
// }
// }
// stage('Releasing package...') {
// steps {
// echo 'Releasing package...'
// // sh 'rm -rf $HOME/www/*'
// // sh 'cp -a ./src/doc/build/html/* $HOME/www/'
// }
// }
stage('Cleaning up...') {
steps {
echo 'Cleaning artifacts...'
sh '/bin/bash -c "make clean"'
sh './scripts/uninstall_fiji.bash'
// sh 'rm -rf ./install_resources'
// sh 'cd ./src/doc && make clean'
}

34
catalog.py
View File

@ -2,16 +2,44 @@
import json
class DacMetadata(object):
"""Represents a display_and_comments.txt metadata file"""
def __init__(self, dac_id, dac_file_path):
"""Contructor.
:param str dac_id: eg "res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1"
:param str dac_file_path: eg "/Users/graffy/ownCloud/ipr/lipase/raw-images/res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/display_and_comments.txt"
"""
self.dac_id = dac_id
self.dac_file_path = dac_file_path
with open(dac_file_path, "r") as dac_file:
self.dac = json.load(dac_file, encoding='latin-1') # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments)
def get_channel_name(self, channel_index):
channels = self.dac['Channels']
return channels[channel_index]["Name"]
class Sequence(object):
def __init__(self, catalog, sequence_id, micro_manager_metadata_file_path):
self.catalog = catalog
self.sequence_id = sequence_id
self.micro_manager_metadata_file_path = micro_manager_metadata_file_path
print(micro_manager_metadata_file_path)
print('reading micro manager metatdata from %s' % micro_manager_metadata_file_path)
# note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments)
with open(micro_manager_metadata_file_path, "r") as mmm_file:
self.mmm = json.load(mmm_file, encoding='latin-1') # note : the micromanager metadata files are encoded in latin-1, not utf8 (see accents in comments)
micro_manager_metadata_file_parent_path = '/'.join(micro_manager_metadata_file_path.split('/')[0:-2])
print('micro_manager_metadata_file_parent_path = %s' % micro_manager_metadata_file_parent_path)
dac_file_path = micro_manager_metadata_file_parent_path + '/display_and_comments.txt'
dac_id = '/'.join(self.sequence_id.split('/')[0:-1])
self.dac = DacMetadata(dac_id, dac_file_path)
for channel_index in range(self.num_channels):
assert self.get_channel_name(channel_index) == self.dac.get_channel_name(channel_index), 'mismatched channel names for channel index %d : %s in %s, %s in %s' % (channel_index, self.get_channel_name(channel_index), self.micro_manager_metadata_file_path, self.dac.get_channel_name(channel_index), self.dac.dac_file_path)
@property
def num_frames(self):
summary = self.mmm['Summary']
@ -65,6 +93,10 @@ class Sequence(object):
channel_index = summary['ChNames'].index(channel_id)
return channel_index
def get_channel_name(self, channel_index):
summary = self.mmm['Summary']
return summary['ChNames'][channel_index]
def get_black(self):
''' returns the black sequence related to the the sequence self

8
hello.cpp
View File

@ -1,8 +0,0 @@
#include <iostream>
using namespace std;
int main(int argc, char * argv[])
{
cout << "hello, world!" << endl;
}

13
lipase.py
View File

@ -1,11 +1,5 @@
# note: fiji's jython doesn't support encoding keyword
# String(label="Please enter your name",description="Name field") name
# OUTPUT String greeting
import sys
print(sys.version) # prints python version
sys.path.append('/Users/graffy/ownCloud/ipr/lipase/lipase.git') # necessary if run from fiji as script otherwise jython fails to find lipase's modules such as catalog
# encoding: utf8
# A Jython script with parameters.
# It is the duty of the scripting framework to harvest
# the 'name' parameter from the user, and then display
@ -18,6 +12,7 @@ from ij.plugin import ImageCalculator
from ij.plugin import ZProjector
from ij import WindowManager
from catalog import Sequence, ImageCatalog
import preprocessing
# greeting = "Hello, " + name + "!"
@ -183,6 +178,9 @@ class Lipase(object):
background_image.show()
return background_image
def estimate_white(self, sequence, channel_id):
return preprocessing.estimate_white([sequence], [channel_id], dark=None)
def process_sequence(self, sequence_id):
'''
:param str sequence_id:
@ -210,6 +208,7 @@ def test_find_white():
depth_index = lipase.find_depth_index(src_image, white_seq)
def run_script():
test_find_white()

271
preprocessing.py Normal file
View File

@ -0,0 +1,271 @@
"""preprocessing of synchrotron images based on telemosToolbox."""
from ij import IJ
from ij.plugin import ImageCalculator
from catalog import ImageCatalog, Sequence
def compute_max(images_file_path):
"""Computes for each pixel position the maximum at this position in all input images.
:param list(str) images_file_path:
:rtype ImagePlus:
"""
assert len(images_file_path) > 1
max_image = IJ.openImage(images_file_path[0])
print('max_image', max_image)
for image_file_path in images_file_path[2:-1]:
other_image = IJ.openImage(image_file_path)
print('other_image', other_image)
ic = ImageCalculator()
ic.run("max", max_image, other_image)
print('max_image', max_image)
return max_image
def replace_outer_frame(image, band_width):
"""Overwrites the outer band of the image by duplicating the value of the pixels that touch this outer band
:param ImagePlus image:
:param int band_width: width of the outer band, in pixels
:rtype ImagePlus:
adaptation for the following code from matlab telemosToolbx's estimatedwhiteFluoImageTelemos function
outer = white_estimate;
white_estimate = white_estimate(4:(end-3),4:(end-3));
outer(1:3,4:(end-3))=repmat(white_estimate(1,:),3,1); # top
outer((end-2):end,4:(end-3))=repmat(white_estimate(end,:),3,1); # bottom
outer(:,1:3)=repmat(outer(:,4),1,3); # left
outer(:,(end-2):end)=repmat(outer(:,end-3),1,3); # right
white_estimate = outer;
clear outer;
"""
raise NotImplementedError()
return image
def estimate_white(sequences, channel_ids, dark=None):
"""Estimation of the white fluorescence image shape of synchrotron light from experimental images of Telemos microscope.
:param list(Sequence) sequences:
:param list(str) channel_ids:
:param ImagePlus or None dark:
:rtype ImagePlus:
Code adapted from matlab telemosToolbx's estimatedwhiteFluoImageTelemos function
% this function is specific of Telemos images (DISCO line SOLEIL)
% acquired using micromanager software linked to imageJ
%% input
% nothing : interactive function
%% output
% nothing : the final reference fluorescence image is saved on disk
%% principe
% NB: MICROMANAGER save images in a structured file folder architecture :
%
% root folder : name given by the user
%
% subfolders roi(n)_tile1 : n folders for each selected roi
% or pos(n) : n folders for each selected position
%
% display_and_comments.txt : file describing the channels acquired
%
% in each subfolder roi(n)_tile1 or Pos(n) :
% images files with name img_00000000(n)_DM300_327-353_00(p).tif
% n = number of time
% p = z focal plane
% DM300_327-353 = name of the channel
% metadata.txt : files describing all metadata associated with the
% acquisition : x, y, z position, camera settings .... etc.
%
% (Nb: depending on the date of acquisition, an extension fluo is found or
% not in the name of fluorescence image. This extension is given in
% contrast to the visible image.)
%
% IMAGES : fluorescence images are considered
%
%
% IMPORTANT:
% it is expected that in the folders at least several images cover the whole field of view
% illuminated by the synchrtron light
% IF IT IS NOT THE CASE, IT WON'T WORK
%
% BASIC: the estimated white image is the MAX of the selected images
% when interactive mode is selected, only first time and first z are
% proposed
% for automatic computing, all images are taken into account
%
% WHAT IS DONE:
% 0 - the user chosse the channels that are considered in the estimation of the
% white fluorescence image
%
% 1 - spurious pixels (three lines and columns around the image) are not taken
% in filtering and replaced by line and column number 4 and end-3
%
% 2 - Filtering
% the estimated dark image is filtered :
% opening to remove white regions brighter than the low frequency signal that should correspond to synchrotron light shape (should be small)
% closing to remove black regions that remains (should be small to avoid synchrotron light shape deformation)
% average filtering
% the image computed should be used to
% - to find the best white z plane from the reference white stack
% acquired for ex. using the so called "Matthieu lame"%
% - correct images for intensities
%% use
% estimatedwhiteFluoImageTelemos
%% Comments
% adapted from proposals 20161050 and 20171187
%% Author
% MF Devaux
% INRA BIA
% PVPP
%% date
% 5 octobre 2017:
% 23 mars 2018
% 3 septembre 2018 : comments and general case
% 16 avril 2019: name of function and default filtering values
% 29 avril 2019 : new comments and spurious pixels
% 27 mai 2019 : offset dark
% 4 juin 2019 : replace exist by isfolder or isfile
% 14 juin : close figure at the end
"""
images_file_path = []
for sequence in sequences:
for channel_id in channel_ids:
channel_index = sequence.get_channel_index(channel_id)
for frame_index in range(sequence.num_frames):
for z_index in range(sequence.num_slices):
images_file_path.append(sequence.get_image_file_path(channel_index, frame_index, z_index))
white_estimate = compute_max(images_file_path)
# modifiy spurious pixels on the side of the images
try:
replace_outer_frame(white_estimate, 3)
except NotImplementedError as error:
print('warning: replace_outer_frame is not implemented yet')
return white_estimate
def find_white_reference_image(white_estimate, white_z_stack):
"""Find the white reference Image among a z stack of reference image the most correlated to the white image estimated from a serie of acquisition.
:param Image white_estimate:
:param Sequence white_z_stack:
% this function is specific of Telemos images (DISCO line SOLEIL)
% acquired using micromanager software linked to imageJ
%% input
% nothing : interactive function
%% output
% zmax : focal plane
%% principe
% images were acquired for a given roi which position is found in
% metadata.txt file
% white and dark images were recorded for the full field of view of the DISCO
% TELEMOS camera
%
% correlation between a estimated white fluorescence image estalished from
% actual acquisitions of a given sample and all z plane acquired for the reference TELEMOS white
% image (Matthieu slide or any fluorescent homogeneous image) :
% The estimated white fluorescence image is generallly obtained bt using function whiteFluoImageTelemosestimation
% This is not compulsary as any homogenous sample image hat can roughly show the shape of illumination can be used to find
% the white reference image
%
%
% the z plane for which the maximum correlation is observed between estimated white and reference white images is retained.
% the white image is then offsetted (its corresponding Dark is subtracted) and copied in the subfolder <WhiteReference> of the sample
% rootfolder to show that it has been specifically selected for the considered experiment
%The matlab corrcoeff function is used
%
% correlation coefficients are saved in a file called
% 'corr.txt' in the subfolder 'WhiteReference'
%
%
% expected input folder hierarchy:
%
% >sampleFolder
% > PosFolders or RoiFolders
% > WhiteEstimate
% >darkFolder
% >darkFolder.smooth
% >whiteFolder
% > darkFolderforWhite
% > darkFolderforWhite.smooth
% >whiteFolder.smooth
%
%
% expected output folder hierarchy:
%
% >sampleFolder
% > PosFolders or RoiFolders
% > WhiteEstimate
% > WhiteReference
% > white after offset
% >darkFolder
% >darkFolder.smooth
% >whiteFolder
% > darkFolderforWhite
% > darkFolderforWhite.smooth
% >whiteFolder.smooth
%% use
% [zmax]=findWhiteReferenceImageTelemos
%% Comments
% written for proposal 20161050
% adapted for proposal 20170043
%% Author
% MF Devaux
% INRA BIA
% PVPP
%% date
% 5 octobre 2017:
% 15 decembre 2017 : adapted to take into account the roi known from
% metadata
% 27 fevrier 2018 : comments details
% 4 septembre 2018: comments and check and naming of folders
% 12 mars 2019 : save white reference with the same size as white estimate
% 16 avril 2019 : include diagonals to check the relevance of white
% reference
% 20 mai 2019 : track folder
% 27 mai 2019 : offset dark
% 4 juin 2019 : replace exist by isfolder or isfile
"""
raise NotImplementedError()
def test_preprocessing():
"""Test preprocessing."""
catalog = ImageCatalog('/Users/graffy/ownCloud/ipr/lipase/raw-images')
sequence = catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos2']
white_estimate = estimate_white([sequence], ['DM300_327-353_fluo'])
find_white_reference_image(white_estimate, sequence.get_white())
if __name__ == '__main__':
test_preprocessing()

2
test1.bash → scripts/install_fiji.bash
View File

@ -2,5 +2,3 @@
wget 'https://downloads.imagej.net/fiji/latest/fiji-linux64.zip'
unzip './fiji-linux64.zip'
rm './fiji-linux64.zip'
./Fiji.app/ImageJ-linux64 --ij2 --headless --run './lipase.py'

2
scripts/uninstall_fiji.bash Executable file
View File

@ -0,0 +1,2 @@
#!/bin/bash
rm -R './Fiji.app'

3
tests/test0001.bash Executable file
View File

@ -0,0 +1,3 @@
./Fiji.app/ImageJ-linux64 --ij2 --headless --run './tests/test0001.py' "raw_images_root_path='/opt/ipr/cluster/work.global/graffy/jenkins-store/lipase/raw-images'"
# on macosx : /Applications/Fiji.app/Contents/MacOS/ImageJ-macosx --ij2 --headless --run './test0001.py'

29
tests/test0001.py Normal file
View File

@ -0,0 +1,29 @@
"""This script is supposed to be launched from fiji's jython interpreter
"""
# # note: fiji's jython doesn't support encoding keyword
#https://imagej.net/Scripting_Headless
#@String raw_images_root_path
# String(label="Please enter your name",description="Name field") name
# OUTPUT String greeting
import sys
print('python version %s' % sys.version) # prints python version
sys.path.append('/Users/graffy/ownCloud/ipr/lipase/lipase.git') # necessary if run from fiji as script otherwise jython fails to find lipase's modules such as catalog
from ij import IJ
from lipase import Lipase, ImageLogger
from catalog import ImageCatalog
def run_script():
catalog = ImageCatalog(raw_images_root_path) # eg '/Users/graffy/ownCloud/ipr/lipase/raw-images'
lipase = Lipase(catalog, debugger=ImageLogger())
sequence = catalog.sequences['res_soleil2018/GGH/GGH_2018_cin2_phiG_I_327_vis_-40_1/Pos2']
white_estimate = lipase.estimate_white(sequence, 'DM300_327-353_fluo')
print(white_estimate)
IJ.saveAsTiff(white_estimate, './white_estimate.tiff')
print('end')
# note : when launched from fiji, __name__ doesn't have the value "__main__", as when launched from python
run_script()