import os
from typing import Any, Dict, List, Optional, Tuple
import tqdm
from mpi4py import MPI
from mpi4py.MPI import Comm
import numpy as np
import httomo
from httomo import globals
from httomo.data.param_sweep_store import ParamSweepWriter
from httomo.method_wrappers.images import ImagesWrapper
from httomo.method_wrappers.save_intermediate import SaveIntermediateFilesWrapper
from httomo.runner.block_split import BlockSplitter
from httomo.runner.dataset_store_backing import DataSetStoreBacking
from httomo.runner.method_wrapper import MethodWrapper
from httomo.runner.pipeline import Pipeline
from httomo.sweep_runner.param_sweep_block import ParamSweepBlock
from httomo.sweep_runner.side_output_manager import SideOutputManager
from httomo.sweep_runner.stages import NonSweepStage, Stages, SweepStage
from httomo.utils import (
catchtime,
log_exception,
log_rank,
log_once,
search_max_slices_iterative,
)
from httomo.runner.gpu_utils import get_available_gpu_memory, gpumem_cleanup
from httomo.preview import PreviewConfig, PreviewDimConfig
from httomo.runner.dataset_store_interfaces import DataSetSource
PAGANIN_SWEEP_VALUE_THRESHOLD = 50
"""The number of sweep values for the paganin filter above which the sweep results should be
written to an hdf5 file as opposed to kept in RAM."""
[docs]
class ParamSweepRunner:
def __init__(
self,
pipeline: Pipeline,
comm: Comm,
side_output_manager: SideOutputManager = SideOutputManager(),
) -> None:
self._pipeline = pipeline
self._vertical_slices_preview = 5
self._side_output_manager = side_output_manager
self._block: Optional[ParamSweepBlock] = None
self._check_params_for_sweep()
self._stages = self.determine_stages()
self._start_sweep_idx, self._stop_sweep_idx = self._determine_sweep_indices(
comm
)
self._sweep_values = self._stages.sweep.values[
self._start_sweep_idx : self._stop_sweep_idx
]
self._set_image_saver_params()
self._comm = comm
@property
def block(self) -> ParamSweepBlock:
if self._block is None:
raise ValueError("Block from input data has not yet been loaded")
return self._block
def _determine_sweep_indices(self, comm: Comm) -> Tuple[int, int]:
no_of_sweep_vals = len(self._stages.sweep.values)
nprocs = comm.size
rank = comm.rank
start = round((no_of_sweep_vals / nprocs) * rank)
end = round((no_of_sweep_vals / nprocs) * (rank + 1))
return start, end
def _check_params_for_sweep(self):
"""
Check pipeline for the number of parameter sweeps present.
If none are defined, then raise an error. If more than one is defined, then also raise
an error, due to not supporting parameter sweeps over more than one parameter at a
time.
"""
params = [m.config_params for m in self._pipeline]
no_of_sweeps = sum(map(_count_tuple_values, params))
if no_of_sweeps == 0:
err_str = "No parameter sweep detected in pipeline"
log_exception(err_str)
raise ValueError(err_str)
if no_of_sweeps > 1:
err_str = (
"Parameter sweep over more than one parameter detected in pipeline; "
"a sweep over only one parameter at a time is supported"
)
log_exception(err_str)
raise ValueError(err_str)
def _set_image_saver_params(self) -> None:
"""
Set the `offset` and `watermark_vals` parameters for any `ImagesWrapper`
instances in the pipeline.
"""
non_sweep_stages = [self._stages.before_sweep, self._stages.after_sweep]
for non_sweep_stage in non_sweep_stages:
for method in non_sweep_stage.methods:
if isinstance(method, ImagesWrapper):
method["offset"] = self._start_sweep_idx
method["watermark_vals"] = self._sweep_values
[docs]
def determine_stages(self) -> Stages:
"""
Groups methods into "before sweep", "sweep", and "after sweep" stages
"""
before_sweep: List[MethodWrapper] = []
sweep_wrapper: Optional[MethodWrapper] = None
sweep_param_name: Optional[str] = None
sweep_param_vals: Optional[Tuple[Any, ...]] = None
after_sweep: List[MethodWrapper] = []
non_sweep_stage_methods: List[MethodWrapper] = before_sweep
for method in self._pipeline:
params = method.config_params
no_of_sweeps = sum(map(_count_tuple_values, [params]))
if no_of_sweeps == 0:
non_sweep_stage_methods.append(method)
else:
sweep_wrapper = method
sweep_param_name, sweep_param_vals = _get_param_sweep_name_and_vals(
params
)
non_sweep_stage_methods = after_sweep
assert sweep_wrapper is not None
assert sweep_param_name is not None
assert sweep_param_vals is not None
return Stages(
before_sweep=NonSweepStage(before_sweep),
sweep=SweepStage(
method=sweep_wrapper,
param_name=sweep_param_name,
values=sweep_param_vals,
),
after_sweep=NonSweepStage(after_sweep),
)
[docs]
def modify_loader_preview(self):
"""
Modifies loader's preview based on the method present in the pipeline.
In some cases we need to override users given preview, e.g., when Paganin filter is present.
"""
# we call this only to find the global size of the data (not very efficient)
source = self._pipeline.loader.make_data_source(padding=(0, 0))
if source.raw_shape[1] < self._vertical_slices_preview:
err_str = (
"The size of the raw data "
f"{source.raw_shape[1]} is smaller than it is required by the sweep run - "
f"{self._vertical_slices_preview}. "
f"Please consider using a larger input dataset with the sweep run."
)
raise ValueError(err_str)
# before modifying preview we need to check if the block fits the memory
for method in self._pipeline._methods:
if "paganin_filter" in method.method_name and method.sweep:
# This method will change the preview (see more in _slices_to_fit_memory_Paganin).
self._vertical_slices_preview = _slices_to_fit_memory_Paganin(source)
preview_new_start_stop = _preview_modifier(
self._pipeline.loader.preview,
source.raw_shape[1],
self._vertical_slices_preview,
)
self._pipeline.loader.preview = PreviewConfig(
angles=self._pipeline.loader.preview.angles,
detector_y=PreviewDimConfig(
start=preview_new_start_stop[0], stop=preview_new_start_stop[1]
),
detector_x=self._pipeline.loader.preview.detector_x,
)
[docs]
def prepare(self):
"""
Load single block containing small number of sinogram slices in input data
"""
source = self._pipeline.loader.make_data_source(padding=(0, 0))
SINO_SLICING_DIM = 1
log_once(f"Loading data with shape {source.global_shape}")
splitter = BlockSplitter(source, source.global_shape[source.slicing_dim])
dataset_block = splitter[0]
sweep_block = ParamSweepBlock(
data=dataset_block.data,
aux_data=dataset_block.aux_data,
slicing_dim=SINO_SLICING_DIM,
)
self._block = sweep_block
def _pass_min_block_length_to_intermediate_data_wrapper(self, stage: NonSweepStage):
assert self._block is not None
for method in stage.methods:
if isinstance(method, SaveIntermediateFilesWrapper):
method.append_config_params(
{"minimum_block_length": self._block.shape[self._block.slicing_dim]}
)
def _execute_non_sweep_stage(self, stage: NonSweepStage):
assert self._block is not None
self._pass_min_block_length_to_intermediate_data_wrapper(stage)
for method in stage.methods:
log_once(f"Running {method.method_name} ({method.package_name})")
self._side_output_manager.update_params(method)
self._block = method.execute(self._block)
self._side_output_manager.append(method.get_side_output())
[docs]
def execute_before_sweep(self):
"""Execute all methods before the parameter sweep"""
self._execute_non_sweep_stage(self._stages.before_sweep)
[docs]
def execute_after_sweep(self):
"""Execute all methods after the parameter sweep"""
self._execute_non_sweep_stage(self._stages.after_sweep)
[docs]
def execute_sweep(self):
"""Execute all param variations of the same method in the sweep"""
method = self._stages.sweep.method
if (
method.method_name == "paganin_filter"
and len(self._stages.sweep.values) > PAGANIN_SWEEP_VALUE_THRESHOLD
):
writer = ParamSweepWriter(
len(self._sweep_values),
self._comm,
globals.run_out_dir,
DataSetStoreBacking.File,
)
else:
writer = ParamSweepWriter(
len(self._sweep_values),
self._comm,
globals.run_out_dir,
DataSetStoreBacking.RAM,
)
log_once(f"Running {method.method_name} ({method.package_name})")
log_once(" Beginning parameter sweep")
log_once(f" Parameter name: {self._stages.sweep.param_name}")
total_vals_str = (
" Total number of values across all processes: "
f"{len(self._stages.sweep.values)}"
)
log_once(total_vals_str)
log_once(f" Values executed in this process: {len(self._sweep_values)}")
# Redirect tqdm progress bar output to /dev/null, and instead manually write sweep
# progress to logfile within loop
progress = tqdm.tqdm(
iterable=self._sweep_values,
file=open(os.devnull, "w"),
unit="value",
ascii=True,
)
for val in progress:
# Blocks are modified in-place by method wrappers, so a new block must be created
# that contains a copy of the input data to the sweep stage
block = ParamSweepBlock(
data=self.block.data.copy(),
aux_data=self.block.aux_data,
slicing_dim=self.block.slicing_dim,
)
self._side_output_manager.update_params(method)
method.append_config_params({self._stages.sweep.param_name: val})
progress.set_postfix_str(f"{self._stages.sweep.param_name}={val}")
log_once(f" {str(progress)}")
block = method.execute(block)
if len(method.get_side_output().keys()) > 0:
raise ValueError(
"Producing a side output is not supported in parameter sweep methods"
)
writer.write_sweep_result(block)
log_once(" Finished parameter sweep")
reader = writer.make_reader()
self._block = reader.read_sweep_results()
if reader.is_file_based:
reader.finalize()
[docs]
def execute(self):
"""Load input data and execute all stages (before sweep, sweep, after sweep)"""
with catchtime() as t:
log_once(f"See the full log file at: {httomo.globals.run_out_dir}/user.log")
self.modify_loader_preview()
self.prepare()
self.execute_before_sweep()
self.execute_sweep()
log_once(
"Stand by until all processes are finished (some can take longer)."
)
comm = MPI.COMM_WORLD
comm.Barrier()
self.execute_after_sweep()
log_once(f"Pipeline finished. Took {t.elapsed:.3f}s")
def _count_tuple_values(d: Dict[str, Any]) -> int:
return sum(1 for v in d.values() if isinstance(v, tuple))
def _get_param_sweep_name_and_vals(d: Dict[str, Any]) -> Tuple[str, Tuple[Any, ...]]:
param_name: Optional[str] = None
sweep_vals: Optional[Tuple[Any, ...]] = None
for k, v in d.items():
if isinstance(v, tuple):
param_name = k
sweep_vals = v
assert param_name is not None
assert sweep_vals is not None
return param_name, sweep_vals
def _preview_modifier(
preview_old: PreviewConfig, det_y_dim_raw: int, vert_slices: int
) -> List[int]:
"""
Modifies the size of the user-defined preview (expand it or shrink it)
"""
preview_new = [0, det_y_dim_raw]
range_preview = preview_old.detector_y.stop - preview_old.detector_y.start
start_new = preview_old.detector_y.start + range_preview // 2 - vert_slices // 2
stop_new = preview_old.detector_y.stop - range_preview // 2 + vert_slices // 2
if start_new >= 0:
preview_new[0] = start_new
if stop_new < det_y_dim_raw:
preview_new[1] = stop_new
return preview_new
def _slices_to_fit_memory_Paganin(source: DataSetSource) -> int:
"""
Estimating the number of vertical slices that can fit on the device for running the Paganin method.
This function assumes that it is running on the GPU-enabled machine
For the Paganin method, the filter kernel width can vary. Therefore, we aim to use the tallest possible
vertical preview that the current device can accommodate.
If the kernel width exceeds the height of the vertical preview, some deviations are expected between
the sweep-run results and the results obtained from processing the full dataset.
"""
from httomo_backends.methods_database.packages.backends.httomolibgpu.supporting_funcs.prep.phase import (
_calc_memory_bytes_for_slices_paganin_filter,
)
gpumem_cleanup()
available_memory = get_available_gpu_memory(10.0)
available_memory_in_GB = round(available_memory / (1024**3), 2)
memory_str = (
f"The amount of the available GPU memory is {available_memory_in_GB} GB"
)
comm = MPI.COMM_WORLD
log_rank(memory_str, comm=comm)
angles_total = source.aux_data.angles_length
det_X_length = source.chunk_shape[2]
def get_mem_bytes(slices):
try:
shape = (angles_total, slices, det_X_length)
extra_input_copy_bytes = np.prod(shape) * np.float32().itemsize
return (
extra_input_copy_bytes
+ _calc_memory_bytes_for_slices_paganin_filter(
shape, dtype=np.float32()
) # NOTE: default arguments to Paganin estimator are passed. It should be safe as long the padding is kept to the next power of
# 2 to be the default one.
)
except:
return 2**64
finally:
gpumem_cleanup()
slices_max = 3
if comm.rank == 0:
# assuming here that the GPU devices have the same amount of memory
slices_max = search_max_slices_iterative(available_memory, get_mem_bytes)
comm.Barrier()
# Broadcast
if comm.size > 1:
slices_max = comm.bcast(slices_max, root=0)
return slices_max