provide normalization to nabu: insure dataset url can be reused by saving the url

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -488,6 +488,8 @@ class _NabuBaseReconstructor:
             config["output"]["location"], settings.NABU_CFG_FILE_FOLDER
         )
         os.makedirs(nabu_cfg_folders, exist_ok=True)
+
+        # configuration file and nabu_tomwer_serving_hatch must be in the same folder
         serving_hatch_file = os.path.join(
             nabu_cfg_folders, settings.NABU_TOMWER_SERVING_HATCH
         )
@@ -503,40 +505,35 @@ class _NabuBaseReconstructor:
                     "value should be provided in extra)infos for scalar defined manually"
                 )
             else:
-                # save the value to a dedicated path in "nabu_tomwer_serving_hatch"
-                if isinstance(self.scan, HDF5TomoScan):
-                    entry_path = self.scan.entry
-                elif isinstance(self.scan, EDFTomoScan):
-                    entry_path = "entry"
-                else:
-                    raise TypeError
-                with HDF5File(serving_hatch_file, mode="a") as h5f:
-                    serving_hatch_data_path = None
-                    # create a unique dataset path to avoid possible conflicts
-                    while (
-                        serving_hatch_data_path is None
-                        or serving_hatch_data_path in h5f
-                    ):
-                        serving_hatch_data_path = "/".join(
-                            [entry_path, str(uuid.uuid1())]
-                        )
-                    # adapt value to what nabues expects.
-                    array = extra_infos["value"]
-                    if (
-                        isinstance(array, numpy.ndarray) and array.ndim == 1
-                    ) or isinstance(array, (tuple, list)):
-                        dim_1 = self.scan.dim_1
-                        array = numpy.repeat(array, dim_1).reshape(len(array), dim_1)
-                    h5f[serving_hatch_data_path] = array
-                file_path = os.path.join(
-                    settings.NABU_CFG_FILE_FOLDER, settings.NABU_TOMWER_SERVING_HATCH
-                )
-                serving_hatch_url = DataUrl(
-                    file_path=file_path,  # configuration file and nabu_tomwer_serving_hatch are in the same folder
-                    data_path=serving_hatch_data_path,
-                    scheme="silx",
-                )
-                config["preproc"]["sino_normalization_file"] = serving_hatch_url.path()
+                # check if the dataset has already been saved once and if we can reuse it
+                dataset_url = extra_infos.get("dataset_created_by_tomwer", None)
+                if dataset_url is not None:
+                    # if an url exists insure we can access it
+                    dataset_url = DataUrl(path=dataset_url)
+                    if os.path.exists(dataset_url.file_path()):
+                        with HDF5File(
+                            dataset_url.file_path(), mode="r", swmr="True"
+                        ) as h5f:
+                            if dataset_url.data_path() not in h5f:
+                                dataset_url = None
+                    else:
+                        dataset_url = None
+                # if unable toi reuse an existing url them dump the value
+                if dataset_url is None:
+                    value = extra_infos["value"]
+                    if isinstance(value, (tuple, list)):
+                        value = numpy.asarray(value)
+                    dataset_url = dump_normalization_array_for_nabu(
+                        scan=self.scan,
+                        array=value,
+                        output_file=serving_hatch_file,
+                    )
+                    extra_infos.update(
+                        {"dataset_created_by_tomwer": dataset_url.path()}
+                    )
+                    self.scan.intensity_normalization.set_extra_infos(extra_infos)
+
+                config["preproc"]["sino_normalization_file"] = dataset_url.path()
         elif source is INormSource.DATASET:
             url = extra_infos["dataset_url"]
             if not isinstance(url, DataUrl):
@@ -579,3 +576,37 @@ class _NabuBaseReconstructor:
                 # then this mean scan_dir has not been provided
                 pass
         return location
+
+
+def dump_normalization_array_for_nabu(
+    scan: TomwerScanBase, output_file: str, array: numpy.ndarray
+) -> DataUrl:
+    if not isinstance(array, numpy.ndarray):
+        raise TypeError(f"array is expected to be a numpy array and not {type(array)}")
+    # save the value to a dedicated path in "nabu_tomwer_serving_hatch"
+    if isinstance(scan, HDF5TomoScan):
+        entry_path = scan.entry
+    elif isinstance(scan, EDFTomoScan):
+        entry_path = "entry"
+    else:
+        raise TypeError
+    with HDF5File(output_file, mode="a") as h5f:
+        serving_hatch_data_path = None
+        # create a unique dataset path to avoid possible conflicts
+        while serving_hatch_data_path is None or serving_hatch_data_path in h5f:
+            serving_hatch_data_path = "/".join([entry_path, str(uuid.uuid1())])
+        # adapt value to what nabues expects.
+        if (isinstance(array, numpy.ndarray) and array.ndim == 1) or isinstance(
+            array, (tuple, list)
+        ):
+            dim_1 = scan.dim_1
+            array = numpy.repeat(array, dim_1).reshape(len(array), dim_1)
+        h5f[serving_hatch_data_path] = array
+    file_path = os.path.join(
+        settings.NABU_CFG_FILE_FOLDER, settings.NABU_TOMWER_SERVING_HATCH
+    )
+    return DataUrl(
+        file_path=file_path,
+        data_path=serving_hatch_data_path,
+        scheme="silx",
+    )