From 42018403258d5d88d98cfd657492889168ffd605 Mon Sep 17 00:00:00 2001
From: Pierre Paleo <pierre.paleo@esrf.fr>
Date: Thu, 28 Nov 2024 11:46:44 +0100
Subject: [PATCH 1/3] pipeline.writer: tell tomoscan to not remove existing
files if a writer has already been initialized
---
nabu/pipeline/writer.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/nabu/pipeline/writer.py b/nabu/pipeline/writer.py
index 3b2e78fc2..e9e671a11 100644
--- a/nabu/pipeline/writer.py
+++ b/nabu/pipeline/writer.py
@@ -1,5 +1,6 @@
from os import path
from tomoscan.esrf import TIFFVolume, MultiTIFFVolume, EDFVolume, JP2KVolume
+from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
from ..utils import check_supported, get_num_threads
from ..resources.logger import LoggerOrPrint
from ..io.writer import NXProcessWriter, HSTVolVolume, NXVolVolume
@@ -113,6 +114,7 @@ class WriterManager:
return vol_writer.data_url.file_path()
def _init_writer(self):
+ self._writer_was_already_initialized = self.extra_options.get("writer_initialized", False)
if self.file_format in ["tiff", "edf", "jp2", "hdf5"]:
writer_kwargs = {
"folder": self.output_dir,
@@ -143,6 +145,11 @@ class WriterManager:
self._h5_entry = self.metadata.get("entry", "entry")
self.writer = self._writer_classes[self.file_format](**writer_kwargs)
self.fname = self.get_fname(self.writer)
+ # In certain cases, tomoscan needs to remove any previous existing volume filess
+ # and avoid calling 'clean_output_data' when writing downstream (for chunk processing)
+ if isinstance(self.writer, VolumeSingleFrameBase):
+ self.writer.skip_existing_data_files_removal = self._writer_was_already_initialized
+ # ---
if path.exists(self.fname):
err = "File already exists: %s" % self.fname
if self.overwrite:
--
GitLab
From 1635299e5764aa7989fb52145a4bda5e3947bda0 Mon Sep 17 00:00:00 2001
From: Pierre Paleo <pierre.paleo@esrf.fr>
Date: Thu, 28 Nov 2024 11:46:50 +0100
Subject: [PATCH 2/3] Adapt chunked.py
---
nabu/pipeline/fullfield/chunked.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/nabu/pipeline/fullfield/chunked.py b/nabu/pipeline/fullfield/chunked.py
index 65f1f6b34..f88221b00 100644
--- a/nabu/pipeline/fullfield/chunked.py
+++ b/nabu/pipeline/fullfield/chunked.py
@@ -668,7 +668,10 @@ class ChunkedPipeline:
"jpeg2000_compression_ratio": options["jpeg2000_compression_ratio"],
"float_clip_values": options["float_clip_values"],
"tiff_single_file": options.get("tiff_single_file", False),
- "single_output_file_initialized": getattr(self.process_config, "single_output_file_initialized", False),
+ "single_output_file_initialized": getattr(
+ self.process_config, "single_output_file_initialized", False
+ ), # COMPAT.
+ "writer_initialized": getattr(self.process_config, "_writer_initialized", False),
"raw_vol_metadata": {"voxelSize": self.dataset_info.pixel_size}, # legacy...
}
writer_extra_options.update(extra_options)
@@ -853,7 +856,8 @@ class ChunkedPipeline:
self.writer.write_data(data)
self.logger.info("Wrote %s" % self.writer.fname)
self._write_histogram()
- self.process_config.single_output_file_initialized = True
+ self.process_config.single_output_file_initialized = True # COMPAT.
+ self.process_config._writer_initialized = True
def _write_histogram(self):
if "histogram" not in self.processing_steps:
--
GitLab
From e83b0dbd47362ada7c934e6bd72b8d423845b599 Mon Sep 17 00:00:00 2001
From: Pierre Paleo <pierre.paleo@esrf.fr>
Date: Thu, 28 Nov 2024 11:49:29 +0100
Subject: [PATCH 3/3] Bump CI
---
doc/v2024_2_0.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/doc/v2024_2_0.md b/doc/v2024_2_0.md
index 27a5678a0..83a69ff6d 100644
--- a/doc/v2024_2_0.md
+++ b/doc/v2024_2_0.md
@@ -48,4 +48,5 @@ This will make a fine addition to our (already large) collection.
- Fix `translation_movements_file` when used alongside projections subsampling
- Fix radios rotation when reconstructing only one slice
- Fix crash when `method = none` in `[reconstruction]` (i.e don't do the reconstruction, only pre-process the data)
- - Many fixes for cone-beam pipeline (see above)
\ No newline at end of file
+ - Many fixes for cone-beam pipeline (see above)
+
--
GitLab