load balancing gpu cpu paganin

correction

transposition

rimesso transpose normale

2by2

compila

piccole corrs

compila OK

piccola corr

problema

comparing with use2by2

finalised

OK

OK

going to pyhst2_2019a

all NRT pass

.gitmodules

+[submodule "rtt"]
+	path = rtt
+	url = https://gitlab.Esrf.Fr/mirone_projects/rtt

PyHST/Cspace/CCspace.h

@@ -606,6 +606,14 @@ int gpu_main(Gpu_Context * self, float *WORK , float * SLICE, int do_preconditio
          int    DETECTOR_DUTY_OVERSAMPLING,
          float *fidelity,
          int npj_offset, int forcePImultipl);
+
+
+int gpu_main_2by2(Gpu_Context * self, float *WORK , float * SLICE, int do_precondition,
+		  int npj_offset, int forcePImultipl ) ;
+
+
+
+
 int dfi_gpu_main(Gpu_Context * self, float *WORK , float * SLICE, int memisonhost);
 
 
@@ -807,6 +815,10 @@ typedef  int (* gpu_main_Symbol     )(Gpu_Context * self, float *WORK , float *
                       int npj_offset, int forcePImultipl) ;
 
 
+typedef  int (* gpu_main_2by2_Symbol     )(Gpu_Context * self, float *WORK , float * SLICE, int precondition,
+                      int npj_offset, int forcePImultipl) ;
+
+
 typedef float ( *gpu_lt_fit_sino_Symbol) (Gpu_Context * self, int Ng, float *gaussian_coeffs,   int   Ns , float * data, int N,
 					  int *csc_II,  int * csc_JJ , float * csc_C ,int csc_nitems_atonce,
 					  int *csr_II,  int * csr_JJ , float * csr_C ,int csr_nitems_atonce,
@@ -887,6 +899,7 @@ struct Gpu_Context_struct {
   int inuse ;
   gpu_lt_fit_sino_Symbol  gpu_lt_fit_sino;
   gpu_main_Symbol  gpu_main;
+  gpu_main_2by2_Symbol  gpu_main_2by2;
   dfi_gpu_main_Symbol  dfi_gpu_main;
   gpu_main_conicity_Symbol  gpu_main_conicity;
   pro_gpu_main_conicity_Symbol  pro_gpu_main_conicity;
@@ -919,8 +932,9 @@ struct Gpu_Context_struct {
   fcomplex  *dev_iWork;
   fcomplex  *dev_iWork_copy;
   float *dev_Filter;
-  int planR2C,planC2R;
+  int planR2C,planC2R, planC2C_2by2;
   void  * a_Proje_voidptr;
+  void  * a_cProje_voidptr;
   float *dev_Work_perproje;
   float *d_work ;
   float* dev_rWork;
@@ -1124,7 +1138,8 @@ struct Gpu_pag_Context_struct {
   void * FFTplan_ptr;
 
   int gpu_pagCtx_initialised ; 
-  int NBunches_todo; 
+  int NBunches_todo;
+  int dones[64000];
   
 }   ;
 
@@ -1190,6 +1205,7 @@ struct CCspace_struct{
   sem_t fbp_sem;
   sem_t gpustat_sem;
   sem_t gpustat_pag_sem;
+  sem_t gpudones_pag_sem;
   sem_t gpustat_med_sem;
   sem_t islicetracker_sem;
   sem_t proupdate_sem;
@@ -1278,13 +1294,14 @@ void CCspace_set_nchunks( CCspace * self   ,int   nchunks ) ;
 #define ACQUIRED        2
 
 void       Paganin(CCspace *  self,  float * Rawptr,
-           int Pos0, int  Pos1, int  Size0, int  Size1,
-           int pos0, int  pos1, int  size0, int  size1,
-           Cparameters *P , int ncpus,   sem_t* fftw_sem,
-           int pstart, int pend, int poffset,
+		   int Pos0, int  Pos1, int  Size0, int  Size1,
+		   int pos0, int  pos1, int  size0, int  size1,
+		   Cparameters *P , int ncpus,   sem_t* fftw_sem,
+		   int pstart, int pend, int poffset,
 		   int p_num_offset,
 		   int mystart,
-		   int npbunches) ;
+		   int npbunches,
+		   int ibunch) ;
 
 /* for ccd_filter */
 #define CCD_FILTER_NONE_ID       0
@@ -1384,7 +1401,7 @@ void  nlm_driver(CCspace *self,int dim0,int dim1,float  *img,float  *result , fl
 float rec_driver(CCspace *self,  float **WORK,float *WORKbis,float *SLICEcalc, int num_bins, int dim_fft,
          float * dumf, fcomplex *dumfC , float *WORK_perproje,float *OVERSAMPLE,int oversampling,
          float * data, int ncpus,float cpu_offset_x,float  cpu_offset_y, int preco, int is_tomolocal,
-         float *fidelity, int npj_offset);
+		 float *fidelity, int npj_offset, int do_2by2);
 
 // Typedef for plugins
 typedef float (*rec_driver_func)(CCspace*,  float **,float *,float *, int, int,

PyHST/Cspace/pyhst_nnfbp.c

@@ -208,7 +208,7 @@ void nnfbp_train(CCspace *self,int dim_fft,int num_bins, float *data_orig, int d
         rec_driver(self,  WORK,WORKbis,SLICEcalc,  num_bins,  dim_fft,
                    dumf, dumfC , WORK_perproje,OVERSAMPLE, oversampling,
                    SINOGRAMMA , ncpus   , cpu_offset_x, cpu_offset_y  ,
-                   0 ,1 , NULL,0);
+                   0 ,1 , NULL,0,0);
         //   sprintf(nomeout,"%s_nnfbpTEST_%04d.edf",self->params.OUTPUT_FILE_ORIG, cW);
         // write_data_to_edf(SLICEcalc , dimslice, dimslice  , nomeout);
 
@@ -316,7 +316,7 @@ void nnfbp_reconstruct(CCspace *self,int dim_fft,int num_bins, float *data_orig,
         rec_driver(self,  WORK,WORKbis,SLICEcalc,  num_bins,  dim_fft,
                    dumf, dumfC , WORK_perproje,OVERSAMPLE, oversampling,
                    SINOGRAMMA , ncpus   , cpu_offset_x, cpu_offset_y  ,
-                   0 ,1 , NULL,0);
+                   0 ,1 , NULL,0,0);
         // Add weighted sigmoid of backprojection with offset to reconstruction
         for( i=0; i< dimslice*dimslice ; i++) {
             SLICE[i]+=self->params.NEURAL_WEIGHTS[ivolta]/(1+exp(self->params.NEURAL_OFFSETS[ivolta]-SLICEcalc[i]));

PyHST/Parameters_module.py

@@ -2708,7 +2708,7 @@ if  (sys.argv[0][-12:]!="sphinx-build")  and not P.DO_V3D_UNSHARP  :
     P.CONICITY_MARGIN_UP   =  numpy.zeros( len( P.first_slices )  ,"i") 
 
     # P.corr_slice_slicedect = numpy.arange(  P.START_VOXEL_3 -1, P.END_VOXEL_3-1 +1, 1)
-    CHANGEME = 10000
+    CHANGEME = 20000
     P.corr_slice_slicedect = numpy.arange(  0, P.SOURCE_Y+P.END_VOXEL_3-1 +1   + CHANGEME, 1)
     P.VOXEL_SIZE = P.IMAGE_PIXEL_SIZE_1  
 

PyHST/PyHST.py

@@ -101,6 +101,7 @@ import glob
 import mpi4py.MPI as MPI   ### strange but I need to have mpi4py already loaded before fabio or on infiniband it crashes
 
 import fabio
+import tempfile
 
 
 
@@ -425,7 +426,6 @@ if(sys.argv[0][-12:]!="sphinx-build"):
     if P.MULTI_PAGANIN_PARS is None:
         callpyhst( inputfile )
     else:
-        from . import tempfile
         steps = [1,1,1,1,1,1,1]
 
         files_to_cancel = []

PyHST/__init__.py

@@ -30,5 +30,5 @@
 # is a problem for you.
 #############################################################################*/
 
-version = "2018b"
+version = "2019a"
 

PyHST/test/test_crayon.py

@@ -196,7 +196,7 @@ def class_provider ( testcase,case_subdir , skip_check=False  ) :
             if self.my_testcase =="nslino":
                 LAUNCHING_INSTRUCTION  = "%s  input.par  " %PROJECT
             if self.my_testcase =="nslinogpu":
-                LAUNCHING_INSTRUCTION  = "%s  input.par  nslino,0" %PROJECT
+                LAUNCHING_INSTRUCTION  = "%s  input.par  scisoft12,0" %PROJECT
                 
             if self.my_testcase =="achille":
                 LAUNCHING_INSTRUCTION  = "%s  input.par  " %PROJECT
@@ -232,6 +232,9 @@ def class_provider ( testcase,case_subdir , skip_check=False  ) :
 
             savedPath = os.getcwd()
             os.chdir( direttorio_output)
+
+            print(LAUNCHING_INSTRUCTION)
+            
             os.system( LAUNCHING_INSTRUCTION )
             os.system("sync")
             # os.system("sleep 60")
@@ -288,17 +291,20 @@ def class_provider ( testcase,case_subdir , skip_check=False  ) :
 def test_classes(testcase):
     myclassi={}
     casi = {
+        # "CRAYON/TESTS/"                 : ( False     , 0,[ "pag_una"]),
         # "CRAYON/TESTS/"                 : ( False     , 0,[ "abs_solo_tutte", "abs_solo_unaslice", "pag_molte_edf", "pag_tutte", "pag_una"]),
         # "HELICOIDAL/TESTS/"              : ( False      , 0,[ "abs_solo_una"]),
         # "ID11_SNOW/TESTS/"              : ( False      , 1,[ "chambolle_pock"]),
+        
         # # "SINO_THRESHOLD/TESTS/"         : ( False     , 1,[ "threshold_unsharp"]),  # provvisoriamente tolta da nslino
-        # # "HEIKKI/TESTS/"                 : ( False   , 1,[ "big"]),  # provvisoriamente tolta da nslino
+        # "HEIKKI/TESTS/"                 : ( False   , 1,[ "big"]),  # provvisoriamente tolta da nslino
+        
         # "MULTIPAGANIN/TESTS/"           : ( False   , 1,[ "volume"]),
         # #  "SIRT_LENA/TESTS/"              : ( True   , 1,[ "oneslice_sirt"]),
         # # "PATCHES_VECTORIAL/TESTS/"      : ( False   , 1,[ "patches", "patches_preconditioned"]),
         # "PATCHES_VECTORIAL/TESTS/"      : ( False   , 1,[  "patches_preconditioned"]),
         # "LENA/TESTS/"                   : ( False   , 1,[ "dictio"]),
-        "LENA_MULTIRINGS/TESTS/"         : ( False   , 1,[ "multiring"]),
+        # "LENA_MULTIRINGS/TESTS/"         : ( False   , 1,[ "multiring"]),
         # # "MOUSSE/TESTS/"                 : ( False   , 1,[ "DL"]),
         # # "NANOPOINTS/TESTS/"             : ( False   , 1,[ "DL"]),
         # "BIG/TESTS/"                    : ( False   , 1,[ "pag_tutte"])

rtt @ 9d33dc24

+Subproject commit 9d33dc24c272f7c991e1021d4e350d19f7d26c82

setup.py

@@ -440,9 +440,8 @@ def do_pyhst():
                             depends=depends,
                            library_dirs= mpilibs_dirs,
                             libraries=["fftw3f_threads", "fftw3f",hdf5_lib, "mpi"],
-                            #extra_compile_args=['-fopenmp'] ,
-                            #extra_link_args=['-fopenmp'] ,
-                            extra_compile_args={'gcc': ["-fPIC"]},
+                           extra_link_args=['-fopenmp'] ,
+                            extra_compile_args={'gcc': ["-fPIC",'-fopenmp' ]},
                             define_macros=define_macros,
                             include_dirs=[ CUDA['include'], numpy.get_include()] + mpi_dirs + hdf5_dirs )
         return module
@@ -648,7 +647,7 @@ def do_pyhst():
         ext_modules.append(build_segmentation())
     if DOCUDA:
         ext_modules.append(build_gputomo())
-        ext_modules.append(build_gpuunsharp())
+        # ext_modules.append(build_gpuunsharp())
         ext_modules.append(build_projection())
         #ext_modules.append(build_wavelets())
 
@@ -731,6 +730,159 @@ else:
 
 
 
+# What you basically need to do is to override the build_ext command class in your setup.py and register it in the command classes. In your custom impl of build_ext, configure and call cmake to configure and then build the extension modules. Unfortunately, the official docs are rather laconic about how to implement custom distutils commands (see Extending Distutils); I find it much more helpful to study the commands code directly. For example, here is the source code for the build_ext command.
+# Example project
+
+# I have prepared a simple project consisting out of a single C extension foo and a python module spam.eggs:
+
+# so-42585210/
+# ├── spam
+# │   ├── __init__.py  # empty
+# │   ├── eggs.py
+# │   ├── foo.c
+# │   └── foo.h
+# ├── CMakeLists.txt
+# └── setup.py
+
+# Files for testing the setup
+
+# These are just some simple stubs I wrote to test the setup script.
+
+# spam/eggs.py (only for testing the library calls):
+
+# from ctypes import cdll
+# import pathlib
+
+
+# def wrap_bar():
+#     foo = cdll.LoadLibrary(str(pathlib.Path(__file__).with_name('libfoo.dylib')))
+#     return foo.bar()
+
+# spam/foo.c:
+
+# #include "foo.h"
+
+# int bar() {
+#     return 42;
+# }
+
+# spam/foo.h:
+
+# #ifndef __FOO_H__
+# #define __FOO_H__
+
+# int bar();
+
+# #endif
+
+# CMakeLists.txt:
+
+# cmake_minimum_required(VERSION 3.10.1)
+# project(spam)
+# set(src "spam")
+# set(foo_src "spam/foo.c")
+# add_library(foo SHARED ${foo_src})
+
+# Setup script
+
+# This is where the magic happens. Of course, there is a lot of room for improvements - you could pass additional options to CMakeExtension class if you need to (for more info on the extensions, see Building C and C++ Extensions), make the CMake options configurable via setup.cfg by overriding methods initialize_options and finalize_options etc.
+
+# import os
+# import pathlib
+
+# from setuptools import setup, Extension
+# from setuptools.command.build_ext import build_ext as build_ext_orig
+
+
+# class CMakeExtension(Extension):
+
+#     def __init__(self, name):
+#         # don't invoke the original build_ext for this special extension
+#         super().__init__(name, sources=[])
+
+
+# class build_ext(build_ext_orig):
+
+#     def run(self):
+#         for ext in self.extensions:
+#             self.build_cmake(ext)
+#         super().run()
+
+#     def build_cmake(self, ext):
+#         cwd = pathlib.Path().absolute()
+
+#         # these dirs will be created in build_py, so if you don't have
+#         # any python sources to bundle, the dirs will be missing
+#         build_temp = pathlib.Path(self.build_temp)
+#         build_temp.mkdir(parents=True, exist_ok=True)
+#         extdir = pathlib.Path(self.get_ext_fullpath(ext.name))
+#         extdir.mkdir(parents=True, exist_ok=True)
+
+#         # example of cmake args
+#         config = 'Debug' if self.debug else 'Release'
+#         cmake_args = [
+#             '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(extdir.parent.absolute()),
+#             '-DCMAKE_BUILD_TYPE=' + config
+#         ]
+
+#         # example of build args
+#         build_args = [
+#             '--config', config,
+#             '--', '-j4'
+#         ]
+
+#         os.chdir(str(build_temp))
+#         self.spawn(['cmake', str(cwd)] + cmake_args)
+#         if not self.dry_run:
+#             self.spawn(['cmake', '--build', '.'] + build_args)
+#         os.chdir(str(cwd))
+
+
+# setup(
+#     name='spam',
+#     version='0.1',
+#     packages=['spam'],
+#     ext_modules=[CMakeExtension('spam/foo')],
+#     cmdclass={
+#         'build_ext': build_ext,
+#     }
+# )
+
+# Testing
+
+# Build the project's wheel, install it. Test the library is installed:
+
+# $ pip show -f spam
+# Name: spam
+# Version: 0.1
+# Summary: UNKNOWN
+# Home-page: UNKNOWN
+# Author: UNKNOWN
+# Author-email: UNKNOWN
+# License: UNKNOWN
+# Location: /Users/hoefling/.virtualenvs/stackoverflow/lib/python3.6/site-packages
+# Requires: 
+# Files:
+#   spam-0.1.dist-info/DESCRIPTION.rst
+#   spam-0.1.dist-info/INSTALLER
+#   spam-0.1.dist-info/METADATA
+#   spam-0.1.dist-info/RECORD
+#   spam-0.1.dist-info/WHEEL
+#   spam-0.1.dist-info/metadata.json
+#   spam-0.1.dist-info/top_level.txt
+#   spam/__init__.py
+#   spam/__pycache__/__init__.cpython-36.pyc
+#   spam/__pycache__/eggs.cpython-36.pyc
+#   spam/eggs.py
+#   spam/libfoo.dylib
+
+# Run the wrapper function from spam.eggs module:
+
+# $ python -c "from spam import eggs; print(eggs.wrap_bar())"
+# 42
+
+
+