From 335bfcd57e59f1699f0e9606210ea3f9eaef295f Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 11:51:59 +0100 Subject: [PATCH 1/8] default parameters injector into output yaml --- httomo/cli.py | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index 580fcd600..d5c22b0dd 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -3,9 +3,10 @@ from os import PathLike from pathlib import Path, PurePath import sys -import tempfile from typing import List, Optional, TextIO, Union, Any +import yaml + import click from mpi4py import MPI from loguru import logger @@ -18,9 +19,10 @@ from httomo.sweep_runner.param_sweep_runner import ParamSweepRunner from httomo.transform_layer import TransformLayer from httomo.utils import log_exception, log_once, mpi_abort_excepthook -from httomo.yaml_checker import validate_yaml_config +from httomo.yaml_checker import validate_yaml_config, _get_template_yaml_conf, PipelineConfig from httomo.runner.task_runner import TaskRunner -from httomo.ui_layer import UiLayer, PipelineFormat +from httomo.ui_layer import UiLayer, PipelineFormat, yaml_loader + try: from . import __version__ @@ -267,13 +269,13 @@ def run( format_enum = ( PipelineFormat.Json if pipeline_format == "Json" else PipelineFormat.Yaml ) - pipeline = generate_pipeline( + pipeline_object = generate_pipeline( in_data_file, pipeline, save_all, method_wrapper_comm, format_enum ) if not does_contain_sweep: execute_high_throughput_run( - pipeline, + pipeline_object, global_comm, gpu_id, max_memory, @@ -283,17 +285,12 @@ def run( save_snapshots, ) else: - execute_sweep_run(pipeline, global_comm) + execute_sweep_run(pipeline_object, global_comm) if mpi_abort_hook: sys.excepthook = sys.__excepthook__ -def _check_yaml(yaml_config: Path, in_data: Path): - """Check a YAML pipeline file for errors.""" - return validate_yaml_config(yaml_config, in_data) - - def transform_limit_str_to_bytes(limit_str: str): try: limit_upper = limit_str.upper() @@ -378,9 +375,13 @@ def initialise_output_directory(pipeline: Union[Path, str]) -> None: # If pipeline is a file path, copy it to output directory if isinstance(pipeline, Path): - with open(pipeline, "r") as input: - pipeline_contents = input.read() - with open(Path(httomo.globals.run_out_dir) / pipeline.name, "a") as output: + pipeline_updated = _substitute_ommitted_default_values(pipeline) + with open(Path(httomo.globals.run_out_dir) / pipeline.name, "w") as file_descriptor: + yaml.dump(pipeline_updated, file_descriptor, default_flow_style=False, sort_keys=False) + # re-open the yaml again in order to add the version comment. + with open(Path(httomo.globals.run_out_dir) / pipeline.name, "r") as input: + pipeline_contents = input.read() + with open(Path(httomo.globals.run_out_dir) / pipeline.name, "w") as output: output.write(f"# Created with HTTomo version {__version__}\n") output.write(pipeline_contents) # If pipeline is a JSON string, write it to a file in the output directory @@ -388,6 +389,19 @@ def initialise_output_directory(pipeline: Union[Path, str]) -> None: with open(httomo.globals.run_out_dir / "pipeline.json", "w") as f: f.write(pipeline) +def _substitute_ommitted_default_values(pipeline: Path) -> PipelineConfig: + pipeline_conf = yaml_loader(pipeline) + templates_conf = _get_template_yaml_conf(pipeline_conf) + for i, (method, template) in enumerate(zip(pipeline_conf, templates_conf)): + template_param_dict = template["parameters"] + method_params = set(method.get("parameters", {}).keys()) + template_params = set(template_param_dict.keys()) + omitted_params = template_params - method_params + + for param in omitted_params: + # insert ommited parameter into the pipeline + pipeline_conf[i]['parameters'][param] = template['parameters'][param] + return pipeline_conf def generate_pipeline( in_data_file: Path, @@ -403,13 +417,13 @@ def generate_pipeline( comm=method_wrapper_comm, pipeline_format=pipeline_format, ) - pipeline = init_UiLayer.build_pipeline() + pipeline_object = init_UiLayer.build_pipeline() # perform transformations on pipeline tr = TransformLayer(comm=method_wrapper_comm, save_all=save_all) - pipeline = tr.transform(pipeline) + pipeline_object = tr.transform(pipeline_object) - return pipeline + return pipeline_object def execute_high_throughput_run( From 3b028718eca41ba719f5e3b77766ea615257e315 Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 11:52:27 +0100 Subject: [PATCH 2/8] default parameters injector into output yaml --- httomo/cli.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index d5c22b0dd..21f30e7ce 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -19,11 +19,14 @@ from httomo.sweep_runner.param_sweep_runner import ParamSweepRunner from httomo.transform_layer import TransformLayer from httomo.utils import log_exception, log_once, mpi_abort_excepthook -from httomo.yaml_checker import validate_yaml_config, _get_template_yaml_conf, PipelineConfig +from httomo.yaml_checker import ( + validate_yaml_config, + _get_template_yaml_conf, + PipelineConfig, +) from httomo.runner.task_runner import TaskRunner from httomo.ui_layer import UiLayer, PipelineFormat, yaml_loader - try: from . import __version__ except: @@ -376,11 +379,18 @@ def initialise_output_directory(pipeline: Union[Path, str]) -> None: # If pipeline is a file path, copy it to output directory if isinstance(pipeline, Path): pipeline_updated = _substitute_ommitted_default_values(pipeline) - with open(Path(httomo.globals.run_out_dir) / pipeline.name, "w") as file_descriptor: - yaml.dump(pipeline_updated, file_descriptor, default_flow_style=False, sort_keys=False) + with open( + Path(httomo.globals.run_out_dir) / pipeline.name, "w" + ) as file_descriptor: + yaml.dump( + pipeline_updated, + file_descriptor, + default_flow_style=False, + sort_keys=False, + ) # re-open the yaml again in order to add the version comment. with open(Path(httomo.globals.run_out_dir) / pipeline.name, "r") as input: - pipeline_contents = input.read() + pipeline_contents = input.read() with open(Path(httomo.globals.run_out_dir) / pipeline.name, "w") as output: output.write(f"# Created with HTTomo version {__version__}\n") output.write(pipeline_contents) @@ -389,6 +399,7 @@ def initialise_output_directory(pipeline: Union[Path, str]) -> None: with open(httomo.globals.run_out_dir / "pipeline.json", "w") as f: f.write(pipeline) + def _substitute_ommitted_default_values(pipeline: Path) -> PipelineConfig: pipeline_conf = yaml_loader(pipeline) templates_conf = _get_template_yaml_conf(pipeline_conf) @@ -400,9 +411,10 @@ def _substitute_ommitted_default_values(pipeline: Path) -> PipelineConfig: for param in omitted_params: # insert ommited parameter into the pipeline - pipeline_conf[i]['parameters'][param] = template['parameters'][param] + pipeline_conf[i]["parameters"][param] = template["parameters"][param] return pipeline_conf + def generate_pipeline( in_data_file: Path, pipeline: Union[Path, str], From a43340da9b3a39d40277aa4d122c7e2e1163cba1 Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 12:24:19 +0100 Subject: [PATCH 3/8] workaround for the issue around sweep aliases --- httomo/cli.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index 21f30e7ce..4938f9885 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -264,7 +264,7 @@ def run( method_wrapper_comm = global_comm if not does_contain_sweep else MPI.COMM_SELF if global_comm.rank == 0: - initialise_output_directory(pipeline) + initialise_output_directory(pipeline, does_contain_sweep) setup_logger(Path(httomo.globals.run_out_dir)) @@ -369,7 +369,7 @@ def set_global_constants( httomo.globals.MAX_CPU_SLICES = max_cpu_slices -def initialise_output_directory(pipeline: Union[Path, str]) -> None: +def initialise_output_directory(pipeline: Union[Path, str], does_contain_sweep: bool) -> None: try: Path.mkdir(httomo.globals.run_out_dir, parents=True, exist_ok=True) except PermissionError as e: @@ -378,20 +378,23 @@ def initialise_output_directory(pipeline: Union[Path, str]) -> None: # If pipeline is a file path, copy it to output directory if isinstance(pipeline, Path): - pipeline_updated = _substitute_ommitted_default_values(pipeline) - with open( - Path(httomo.globals.run_out_dir) / pipeline.name, "w" - ) as file_descriptor: - yaml.dump( - pipeline_updated, - file_descriptor, - default_flow_style=False, - sort_keys=False, - ) - # re-open the yaml again in order to add the version comment. - with open(Path(httomo.globals.run_out_dir) / pipeline.name, "r") as input: + path_to_pipeline = pipeline + path_to_saved_pipeline = Path(httomo.globals.run_out_dir) / pipeline.name + # if does_contain_sweep do not inject default parameters due to issue around "sweep" aliases in yaml + if not does_contain_sweep: + path_to_pipeline = path_to_saved_pipeline + pipeline_updated = _substitute_ommitted_default_values(pipeline) + with open(path_to_saved_pipeline, "w" + ) as file_descriptor: + yaml.dump( + pipeline_updated, + file_descriptor, + default_flow_style=False, + sort_keys=False, + ) + with open(path_to_pipeline, "r") as input: pipeline_contents = input.read() - with open(Path(httomo.globals.run_out_dir) / pipeline.name, "w") as output: + with open(path_to_saved_pipeline, "w") as output: output.write(f"# Created with HTTomo version {__version__}\n") output.write(pipeline_contents) # If pipeline is a JSON string, write it to a file in the output directory From 561417c37c88386794c588f8281e72208696d44c Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 12:38:20 +0100 Subject: [PATCH 4/8] fix for tests --- httomo/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httomo/cli.py b/httomo/cli.py index 4938f9885..a0a1693e1 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -369,7 +369,7 @@ def set_global_constants( httomo.globals.MAX_CPU_SLICES = max_cpu_slices -def initialise_output_directory(pipeline: Union[Path, str], does_contain_sweep: bool) -> None: +def initialise_output_directory(pipeline: Union[Path, str], does_contain_sweep: Optional[bool] = False) -> None: try: Path.mkdir(httomo.globals.run_out_dir, parents=True, exist_ok=True) except PermissionError as e: From bb0b46f8bd6b875acbcfc09070f04ad0cbc5b9dc Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 12:39:26 +0100 Subject: [PATCH 5/8] linting --- httomo/cli.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index a0a1693e1..0311d65c0 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -369,7 +369,9 @@ def set_global_constants( httomo.globals.MAX_CPU_SLICES = max_cpu_slices -def initialise_output_directory(pipeline: Union[Path, str], does_contain_sweep: Optional[bool] = False) -> None: +def initialise_output_directory( + pipeline: Union[Path, str], does_contain_sweep: Optional[bool] = False +) -> None: try: Path.mkdir(httomo.globals.run_out_dir, parents=True, exist_ok=True) except PermissionError as e: @@ -380,12 +382,11 @@ def initialise_output_directory(pipeline: Union[Path, str], does_contain_sweep: if isinstance(pipeline, Path): path_to_pipeline = pipeline path_to_saved_pipeline = Path(httomo.globals.run_out_dir) / pipeline.name - # if does_contain_sweep do not inject default parameters due to issue around "sweep" aliases in yaml + # if does_contain_sweep do not inject default parameters due to issue around "sweep" aliases in yaml if not does_contain_sweep: path_to_pipeline = path_to_saved_pipeline - pipeline_updated = _substitute_ommitted_default_values(pipeline) - with open(path_to_saved_pipeline, "w" - ) as file_descriptor: + pipeline_updated = _substitute_ommitted_default_values(pipeline) + with open(path_to_saved_pipeline, "w") as file_descriptor: yaml.dump( pipeline_updated, file_descriptor, From a8f8b5d54e3b1406b2c6d7aba2b4a33589f620f3 Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 14:49:37 +0100 Subject: [PATCH 6/8] adding copying of distortion coeff file --- httomo/cli.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index 0311d65c0..b43415884 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -8,6 +8,7 @@ import yaml import click +import shutil from mpi4py import MPI from loguru import logger @@ -380,12 +381,16 @@ def initialise_output_directory( # If pipeline is a file path, copy it to output directory if isinstance(pipeline, Path): + pipeline_conf = yaml_loader(pipeline) + distortion_coeff_path = _get_distortion_coeff_path(pipeline_conf) + if distortion_coeff_path is not None: + shutil.copyfile(distortion_coeff_path, Path(httomo.globals.run_out_dir) / "dist_coeff.txt") path_to_pipeline = pipeline path_to_saved_pipeline = Path(httomo.globals.run_out_dir) / pipeline.name # if does_contain_sweep do not inject default parameters due to issue around "sweep" aliases in yaml if not does_contain_sweep: path_to_pipeline = path_to_saved_pipeline - pipeline_updated = _substitute_ommitted_default_values(pipeline) + pipeline_updated = _substitute_ommitted_default_values(pipeline_conf) with open(path_to_saved_pipeline, "w") as file_descriptor: yaml.dump( pipeline_updated, @@ -404,8 +409,7 @@ def initialise_output_directory( f.write(pipeline) -def _substitute_ommitted_default_values(pipeline: Path) -> PipelineConfig: - pipeline_conf = yaml_loader(pipeline) +def _substitute_ommitted_default_values(pipeline_conf: PipelineConfig) -> PipelineConfig: templates_conf = _get_template_yaml_conf(pipeline_conf) for i, (method, template) in enumerate(zip(pipeline_conf, templates_conf)): template_param_dict = template["parameters"] @@ -418,6 +422,12 @@ def _substitute_ommitted_default_values(pipeline: Path) -> PipelineConfig: pipeline_conf[i]["parameters"][param] = template["parameters"][param] return pipeline_conf +def _get_distortion_coeff_path(pipeline_conf: PipelineConfig) -> Union[None, Path]: + distortion_coeff_path = None + for method in pipeline_conf: + if 'distortion_correction' in method['method']: + distortion_coeff_path = method['parameters']['metadata_path'] + return distortion_coeff_path def generate_pipeline( in_data_file: Path, From 711b75ec05a1b6e5c57e95346a6bdc8a07e36371 Mon Sep 17 00:00:00 2001 From: algol Date: Mon, 20 Apr 2026 14:50:47 +0100 Subject: [PATCH 7/8] adding copying of distortion coeff file --- httomo/cli.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index b43415884..d821917fa 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -384,7 +384,10 @@ def initialise_output_directory( pipeline_conf = yaml_loader(pipeline) distortion_coeff_path = _get_distortion_coeff_path(pipeline_conf) if distortion_coeff_path is not None: - shutil.copyfile(distortion_coeff_path, Path(httomo.globals.run_out_dir) / "dist_coeff.txt") + shutil.copyfile( + distortion_coeff_path, + Path(httomo.globals.run_out_dir) / "dist_coeff.txt", + ) path_to_pipeline = pipeline path_to_saved_pipeline = Path(httomo.globals.run_out_dir) / pipeline.name # if does_contain_sweep do not inject default parameters due to issue around "sweep" aliases in yaml @@ -409,7 +412,9 @@ def initialise_output_directory( f.write(pipeline) -def _substitute_ommitted_default_values(pipeline_conf: PipelineConfig) -> PipelineConfig: +def _substitute_ommitted_default_values( + pipeline_conf: PipelineConfig, +) -> PipelineConfig: templates_conf = _get_template_yaml_conf(pipeline_conf) for i, (method, template) in enumerate(zip(pipeline_conf, templates_conf)): template_param_dict = template["parameters"] @@ -422,13 +427,15 @@ def _substitute_ommitted_default_values(pipeline_conf: PipelineConfig) -> Pipeli pipeline_conf[i]["parameters"][param] = template["parameters"][param] return pipeline_conf + def _get_distortion_coeff_path(pipeline_conf: PipelineConfig) -> Union[None, Path]: distortion_coeff_path = None for method in pipeline_conf: - if 'distortion_correction' in method['method']: - distortion_coeff_path = method['parameters']['metadata_path'] + if "distortion_correction" in method["method"]: + distortion_coeff_path = method["parameters"]["metadata_path"] return distortion_coeff_path + def generate_pipeline( in_data_file: Path, pipeline: Union[Path, str], From 72c347f4940c3e64e37628fa3ab7c0f797298462 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 29 Apr 2026 12:16:27 +0100 Subject: [PATCH 8/8] requested fixes --- httomo/cli.py | 6 +++--- tests/test_cli.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index d821917fa..45f756c6a 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -371,7 +371,7 @@ def set_global_constants( def initialise_output_directory( - pipeline: Union[Path, str], does_contain_sweep: Optional[bool] = False + pipeline: Union[Path, str], does_contain_sweep: bool ) -> None: try: Path.mkdir(httomo.globals.run_out_dir, parents=True, exist_ok=True) @@ -393,7 +393,7 @@ def initialise_output_directory( # if does_contain_sweep do not inject default parameters due to issue around "sweep" aliases in yaml if not does_contain_sweep: path_to_pipeline = path_to_saved_pipeline - pipeline_updated = _substitute_ommitted_default_values(pipeline_conf) + pipeline_updated = _substitute_omitted_default_values(pipeline_conf) with open(path_to_saved_pipeline, "w") as file_descriptor: yaml.dump( pipeline_updated, @@ -412,7 +412,7 @@ def initialise_output_directory( f.write(pipeline) -def _substitute_ommitted_default_values( +def _substitute_omitted_default_values( pipeline_conf: PipelineConfig, ) -> PipelineConfig: templates_conf = _get_template_yaml_conf(pipeline_conf) diff --git a/tests/test_cli.py b/tests/test_cli.py index 27d6bc39f..cd236acb6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -181,7 +181,7 @@ def test_initialise_output_directory_handles_json_string(tmp_path): json_string = json.dumps(SAMPLE_JSON_PIPELINE) # Call the function with a JSON string - initialise_output_directory(json_string) + initialise_output_directory(json_string, False) # Verify directory was created assert output_dir.exists() @@ -206,7 +206,7 @@ def test_initialise_output_directory_handles_path_input( pipeline_path = Path(__file__).parent.parent / standard_loader # Call the function with a Path - initialise_output_directory(pipeline_path) + initialise_output_directory(pipeline_path, False) # Verify directory was created assert output_dir.exists() @@ -219,7 +219,7 @@ def test_output_dir_created_if_doesnt_exist(tmp_path: Path, standard_loader: str output_dir_cli_arg = tmp_path / "out" httomo.globals.run_out_dir = output_dir_cli_arg / "httomo-output-dir" pipeline_path = Path(__file__).parent.parent / standard_loader - initialise_output_directory(pipeline_path) + initialise_output_directory(pipeline_path, False) assert output_dir_cli_arg.exists() assert httomo.globals.run_out_dir.exists()