Skip to content
Snippets Groups Projects

reprocess_xrdp script updates

Merged Thomas Vincent requested to merge update-reprocess-script into main
1 file
+ 54
5
Compare changes
  • Side-by-side
  • Inline
+ 54
5
@@ -4,15 +4,16 @@ Usage:
python scripts/reprocess_xrpd.py --proposal ma5876 \
--samples 02_cell1 03_cell2 \
--pyfai-config /data/visitor/ma5876/id31/20231117/PROCESSED_DATA/calibration_good/20231117_json_good_bottom_beam_q_MM.json \
--run qunits
--run reprocess_q_folder
python scripts/reprocess_xrpd.py --proposal ma5876 \
--samples 02_cell1 03_cell2 \
--pyfai-config /data/visitor/ma5876/id31/20231117/PROCESSED_DATA/calibration_good/20231117_json_good_bottom_beam_2th_MM.json \
--run 2thunits
--run reprocess_2th_folder
"""
import json
import os
import sys
from glob import glob
@@ -39,10 +40,12 @@ def get_parameters(
scan_number,
detector_name,
pyfai_config,
pyfai_method,
energy,
monitor_name=None,
reference_counts=None,
run="",
save_as_ascii=False,
):
if run:
output_filename = dataset_filename.replace("RAW_DATA", f"PROCESSED_DATA/{run}")
@@ -53,6 +56,10 @@ def get_parameters(
output_filename = dataset_filename.replace("RAW_DATA", "PROCESSED_DATA")
convert_destination = dataset_filename.replace("RAW_DATA", "PROCESSED_DATA")
output_filename_template = output_filename.replace(
".h5", f"_{scan_number}_{detector_name}_%04d.xye"
)
convert_destination = convert_destination.replace(
".h5", f"_{scan_number}_{detector_name}.json"
)
@@ -65,7 +72,7 @@ def get_parameters(
{
"task_identifier": "PyFaiConfig",
"name": "integration_options",
"value": {"method": "no_csr_ocl_gpu"},
"value": {"method": pyfai_method},
},
{
"task_identifier": "IntegrateBlissScan",
@@ -94,15 +101,27 @@ def get_parameters(
"value": 1,
},
{
# Use a very long retry timeout to mitigate issues
# with multiple workflows accessing the same files
"task_identifier": "IntegrateBlissScan",
"name": "retry_timeout",
"value": 10,
"value": 6 * 3600, # 6h hours
},
{
"task_identifier": "IntegrateBlissScan",
"name": "detector_name",
"value": detector_name,
},
{
"task_identifier": "SaveNexusPatternsAsAscii",
"name": "output_filename_template",
"value": output_filename_template,
},
{
"task_identifier": "SaveNexusPatternsAsAscii",
"name": "enabled",
"value": save_as_ascii,
},
]
return {"inputs": inputs, "convert_destination": convert_destination}
@@ -168,12 +187,26 @@ def main(argv=None) -> None:
help="PyFAI .poni or .json file",
)
parser.add_argument(
"--pyfai-method",
type=str,
default="full_csr_ocl_gpu",
required=False,
help="PyFAI integrator method",
)
parser.add_argument(
"--local",
action="store_true",
help="Execute workflows in the current environment",
)
parser.add_argument(
"--save-ascii",
action="store_true",
help="Export integrated patterns as text files",
)
parser.add_argument(
"--run",
type=str,
@@ -182,6 +215,12 @@ def main(argv=None) -> None:
help="Processed sub-directory name",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Do not run anything but print which datasets would be processed",
)
args = parser.parse_args(argv[1:])
session_pattern = os.path.join(
@@ -222,13 +261,19 @@ def main(argv=None) -> None:
scan_number,
DETECTOR,
args.pyfai_config,
args.pyfai_method,
energy,
monitor_name=args.monitor,
reference_counts=args.reference,
run=args.run,
save_as_ascii=args.save_ascii,
)
if args.local:
if args.dry_run:
print(f"Would process {dataset_filename}::/{scan_number}.1 with:")
print(json.dumps(kwargs, indent=2, sort_keys=True))
elif args.local:
try:
print(f"Processing {dataset_filename}::/{scan_number}.1 ...")
execute_graph(WORKFLOW, **kwargs)
except Exception as e:
traceback.print_exc()
@@ -236,13 +281,17 @@ def main(argv=None) -> None:
else:
future = submit(args=(WORKFLOW,), kwargs=kwargs)
futures.append((future, dataset_filename, scan_number))
print(f"Submitted {dataset_filename}::/{scan_number}.1")
for future, dataset_filename, scan_number in futures:
try:
future.get()
except Exception as e:
print(f"FAILED {dataset_filename}::/{scan_number}.1")
traceback.print_exc()
failed.append(f"{dataset_filename}::/{scan_number}.1 ({e})")
else:
print(f"COMPLETED {dataset_filename}::/{scan_number}.1")
if failed:
print("FAILED:")
Loading