CDI on scisoft10

.gitlab-ci.yml

-test3:
-  stage: test
-  only:
-    - scisoft12
-  tags:
-    - scisoft12
-  script:
-    - echo 'current dir is '
-    - pwd
-    - unset PYTHONPATH
-    - export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
-    - unset LDFLAGS
-    - unset CPPFLAGS
-    - unset LD_LIBRARY_PATH
-    - rm -rf ~/software/virtualenv_xrstools_python3/
-    - virtualenv --python=python3 ~/software/virtualenv_xrstools_python3/
-    - source ~/software/virtualenv_xrstools_python3/bin/activate
-    - pip install setuptools --upgrade
-    - pip install pip --upgrade
-    - pip install numpy
-    - pip install -r requirements.txt
-    - ln -s  /usr/lib/python3/dist-packages/PyQt* ~/software//virtualenv_xrstools_python3/lib/python3.4/site-packages/
-    - ln -s  /usr/lib/python3/dist-packages/sip* ~/software//virtualenv_xrstools_python3/lib/python3.4/site-packages/
-    - pip install .
-    - cd nonregressions
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
-    - deactivate
-    
-  after_script:
-    - pwd
-    
-test:
-  stage: test
-  only:
-    - scisoft13
-  tags:
-    - scisoft13
-  script:
-    - echo 'current dir is '
-    - pwd
-    - unset PYTHONPATH
-    - export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
-    - unset LDFLAGS
-    - unset CPPFLAGS
-    - unset LD_LIBRARY_PATH
-    - rm -rf ~/software/virtualenv_xrstools_python2/
-    - virtualenv --python=python2 ~/software/virtualenv_xrstools_python2/
-    - source ~/software/virtualenv_xrstools_python2/bin/activate
-    - pip install setuptools --upgrade
-    - pip install pip --upgrade
-    - pip install numpy
-    - pip install -r requirements.txt
-    - ln -s  /usr/lib/python2.7/dist-packages/PyQt* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
-    - ln -s  /usr/lib/python2.7/dist-packages/sip* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
-    - pip install .
-    - cd nonregressions
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
-    - deactivate
-    
-  after_script:
-    - pwd
+stages:
+  - test
+#  - deploy
 
-testnslino:
-  stage: test
-  only:
-    - nslino
-  tags:
-    - nslino
-  script:
-    - echo 'current dir is '
-    - pwd
-    - unset PYTHONPATH
-    - export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
-    - unset LDFLAGS
-    - unset CPPFLAGS
-    - unset LD_LIBRARY_PATH
-    - rm -rf ~/software/virtualenv_xrstools_python2/
-    - virtualenv --python=python2 ~/software/virtualenv_xrstools_python2/
-    - source ~/software/virtualenv_xrstools_python2/bin/activate
-    - pip install setuptools --upgrade
-    - pip install pip --upgrade
-    - pip install numpy
-    - pip install -r requirements.txt
-    - ln -s  /usr/lib/python2.7/dist-packages/PyQt* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
-    - ln -s  /usr/lib/python2.7/dist-packages/sip* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
-    - pip install .
-    - cd nonregressions
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
-    - deactivate
-    
-  after_script:
-    - pwd
+variables:
+  PROJECT_NAME: "xrstools"
+  LOCAL_REPO: "/dev/shm/gitlab_ci/xrstools"
+  VENV_DIR: /dev/shm/gitlab_ci/xrstools/venv
+  USE_CLEAN_VENV: "false"
 
 
 
-test3nslino:
+test:
   stage: test
   only:
-    - nslino3
+    - cdi
   tags:
-    - nslino
-  script:
-    - echo 'current dir is '
-    - pwd
-    - unset PYTHONPATH
-    - export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
-    - unset LDFLAGS
-    - unset CPPFLAGS
-    - unset LD_LIBRARY_PATH
-    - rm -rf ~/software/virtualenv_xrstools_python3/
-    - virtualenv --python=python3 ~/software/virtualenv_xrstools_python3/
-    - source ~/software/virtualenv_xrstools_python3/bin/activate
-    - pip install setuptools --upgrade
-    - pip install pip --upgrade
-    - pip install numpy
+    - scisoft10
+  before_script:
+    - unset HTTP_PROXY
+    - unset HTTPS_PROXY
+    - if [ ! -d $LOCAL_REPO ]; then mkdir -p $LOCAL_REPO; fi
+    - if [ -d $VENV_DIR ]; then if [ "$USE_CLEAN_VENV" == "true" ]; then rm -rf $VENV_DIR; fi; fi
+    - python3 -m venv $VENV_DIR
+    - source $VENV_DIR/bin/activate
+    - echo "Running $(python --version) from $(which python) on $(hostname) ($(arch))"
+
+    - pip install  --upgrade  pip setuptools wheel
+    - pip install --upgrade numpy silx
     - pip install -r requirements.txt
-    - pip install PyQt5
-    - pip install .
+
+  script:
+    - python -m pip install .
     - cd nonregressions
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
+    # - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
     - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
-    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
+    # - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
     - deactivate
     
   after_script:
     - pwd
+    

XRStools/XRS_swissknife.py

@@ -2902,7 +2902,7 @@ def superR_scal_deltaXimages(mydata):
         
     if ('roi_keys' in mydata) :
         u_roi_keys  = mydata['roi_keys']
-        u_roi_keys = [str(t) for t in roi_keys]
+        u_roi_keys = [str(t) for t in u_roi_keys]
         roi_keys = [ t for t in u_roi_keys if t in roi_keys]
 
     roi_keys = [str(t) for t in roi_keys]
@@ -3360,7 +3360,7 @@ def superR_scal_deltaXimages_Esynt(mydata):
         
     if ('roi_keys' in mydata) :
         u_roi_keys  = mydata['roi_keys']
-        # u_roi_keys = [str(t) for t in roi_keys]
+        u_roi_keys = [str(t) for t in u_roi_keys]
         roi_keys = [ t for t in u_roi_keys if t in roi_keys]
 
     roi_keys = [str(t) for t in roi_keys]
@@ -3492,6 +3492,17 @@ def superR_scal_deltaXimages_Esynt(mydata):
     for i,rk in enumerate(roi_keys):
         fattori[rk] = 1.0
 
+    if "load_factors_from" in mydata:
+        loaded_fattori = json.load(  open(mydata["load_factors_from"],"r")  )
+
+        for i,rk in enumerate(roi_keys):
+            if rk in loaded_fattori:
+                print(" FATTORE ", rk , " ", loaded_fattori[rk]  ) 
+                fattori[rk] *= loaded_fattori[rk]
+            else:
+                fattori[rk] = 0
+
+        
     ## These arrays below will contain, after summation and for each process,
     ## the contribution of the process's workload
     ## They will be mpi Reduced to get the final result
@@ -3549,14 +3560,12 @@ def superR_scal_deltaXimages_Esynt(mydata):
             scalDS[iroi,iz] =  scalDS[iroi, iz]+   plane_contrib*fattori[rk]
 
             scalDD[iroi]     += (m*m).sum()
-
-                
+            
     h5f.close()
 
     ##  
     ## ######################
 
-
     ## stuffs for checking (integrated_images)
     if nprocs>1:    
         for n in  list(integrated_images.keys()):

paleo_gitlab.yml

+stages:
+  - test
+  - deploy
+
+
+variables:
+  PROJECT_NAME: "nabu"
+  LOCAL_REPO_SHELL: "/dev/shm/gitlab_ci/nabu"
+  VENV_DIR_SHELL: /dev/shm/gitlab_ci/nabu/venv
+  LOCAL_REPO_DOCKER: "/root/gitlab_ci/nabu"
+  VENV_DIR_DOCKER: "/root/gitlab_ci/nabu/venv"
+  USE_CLEAN_VENV: "false"
+
+
+.linux_base:
+  stage: test
+  before_script:
+    - unset HTTP_PROXY
+    - unset HTTPS_PROXY
+    - if [ ! -d $LOCAL_REPO ]; then mkdir -p $LOCAL_REPO; fi
+    - if [ -d $VENV_DIR ]; then if [ "$USE_CLEAN_VENV" == "true" ]; then rm -rf $VENV_DIR; fi; fi
+    - python3 -m venv $VENV_DIR
+    - source $VENV_DIR/bin/activate
+    - echo "Running $(python --version) from $(which python) on $(hostname) ($(arch))"
+    - pip install --upgrade pip setuptools wheel
+    - pip install --upgrade numpy silx
+    - pip install scipy
+    - if [[ $CI_RUNNER_TAGS == *"gpu"* ]]; then pip install pycuda scikit-cuda; fi
+
+  script:
+    - python -m pip install .
+    - nabu-test
+#  allow_failure: true
+
+
+linux_shell:
+  extends: .linux_base
+  variables:
+    LOCAL_REPO: $LOCAL_REPO_SHELL
+    VENV_DIR: ${VENV_DIR_SHELL}
+  tags:
+    - shell
+
+
+linux_docker:
+  extends: .linux_base
+  image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
+  variables:
+    LOCAL_REPO: $LOCAL_REPO_DOCKER
+    VENV_DIR: $VENV_DIR_DOCKER
+  tags:
+    - docker
+
+
+
+.pages_base:
+  stage: deploy
+  before_script:
+    - source $VENV_DIR/bin/activate
+    - pip install --upgrade pip
+    - pip install sphinx cloud_sptheme nbsphinx recommonmark numpy
+  script:
+    - cd doc
+    - make html
+    - cd ..
+    - mkdir public
+    - cp -rv doc/_build/html/* public/
+  artifacts:
+    paths:
+      - public
+
+
+pages_shell:
+  extends: .pages_base
+  variables:
+    LOCAL_REPO: $LOCAL_REPO_SHELL
+    VENV_DIR: $VENV_DIR_SHELL
+  tags:
+    - shell
+  artifacts:
+    paths:
+    - public
+