Fix merge_hdf5_reconstructions() order wrong for more than 9999 projections

nabu/pipeline/fullfield/local_reconstruction.py

@@ -23,6 +23,10 @@ def get_gpus_ids(resources_cfg):
     return list(range(resources_cfg["gpus"]))
 
 
+def variable_idxlen_sort(fname):
+    return int(fname.split("_")[-1].split(".")[0])
+
+
 class ChunkedReconstructor:
     """
     A class for doing full-volume reconstructions.
@@ -369,7 +373,13 @@ class ChunkedReconstructor:
         out_cfg = self.process_config.nabu_config["output"]
         if files is None:
             files = list(self.results.values())
-        files.sort()
+        try:
+            files.sort(key=variable_idxlen_sort)
+        except:
+            self.logger.error(
+                "Lexical ordering failed, falling back to default sort - it will fail for more than 10k projections"
+            )
+            files.sort()
         local_files = [
             join(out_cfg["file_prefix"], basename(fname))
             for fname in files
@@ -464,7 +474,14 @@ class ChunkedReconstructor:
         #
         h5_path = join(masterfile_entry, *[masterfile_process_name, "results", "data"])
         #
-        files = sorted(self._histograms.values())
+
+        try:
+            files = sorted(self._histograms.values(), key=variable_idxlen_sort)
+        except:
+            self.logger.error(
+                "Lexical ordering of histogram failed, falling back to default sort - it will fail for more than 10k projections"
+            )
+            files = sorted(self._histograms.values())
         data_urls = []
         for fname in files:
             url = DataUrl(
@@ -808,7 +825,9 @@ class GroupedReconstructor(ChunkedReconstructor):
             self.logger.info("Resuming from sinograms in %s" % projections)
         if projections is None:
             self.set_reconstruction_state("radios")
+            # TODO this is clumsy - see comment above the class definition
             LocalReconstruction.reconstruct(self)
+            #
             self._projections = self.merge_hdf5_reconstructions(prefix="sinogram_", merge_histograms=False)
             self.merge_data_dumps(axis=0)
             self._destroy_pipeline()