import sys
import logging
from time import perf_counter_ns
from traceback import format_tb
from typing import Any, Callable, Dict, List, Literal, Tuple
from loguru import logger
from mpi4py import MPI
import numpy as np
from PIL import Image
import os
import httomo.globals
from pathlib import Path
from httomo_backends.methods_database.query import Pattern
gpu_enabled = False
try:
import cupy as xp
try:
xp.cuda.Device(0).compute_capability
gpu_enabled = True # CuPy is installed and GPU is available
except xp.cuda.runtime.CUDARuntimeError:
import numpy as xp
except ImportError:
import numpy as xp
[docs]
def save_2d_snapshot(
data_slice: xp.ndarray, methods_name: str, section_index: int
) -> None:
"""
A utility to save stage snapshots as images to help debugging process
:param data_slice: a 2D array to save as a jpeg image
:type data_slice: xp.ndarray
:param methods_name: the name of the image to be saved (e.g. method's name)
:type methods_name: str
:param section_index: the index of the section
:type section_index: int
"""
output_dir_snapshots = (
Path(httomo.globals.run_out_dir) / "pipeline_stages_snapshots"
)
output_dir_snapshots.mkdir(parents=True, exist_ok=True)
data_slice = np.nan_to_num(data_slice, copy=False, nan=0.0, posinf=0, neginf=0)
vmin, vmax = np.percentile(data_slice, [1, 99])
data_slice = np.clip(data_slice, vmin, vmax)
data_slice = (data_slice - vmin) / (vmax - vmin)
data_slice = (data_slice * 255).astype(np.uint8)
filename = f"{0}{section_index}{methods_name}.{'jpeg'}"
filepath_name = os.path.join(output_dir_snapshots, f"{filename}")
Image.fromarray(data_slice, mode="L").save(filepath_name, quality=95)
[docs]
def log_once(output: Any, level: int = logging.INFO) -> None:
"""
Log output to console and log file if the process' global rank is zero.
Parameters
----------
output : Any
The item to be printed.
level : int, optional
The level of the log message. See
https://docs.python.org/3/library/logging.html#logging-levels.
"""
if MPI.COMM_WORLD.rank == 0:
if isinstance(output, list):
output = "".join([f"{out}" for out in output])
if level == logging.DEBUG:
logger.debug(output)
elif level == logging.WARNING:
logger.warning(output)
else:
# logger.info(output)
if "section" in output:
logger.opt(colors=True).info("<cyan>{}</cyan>".format(output))
elif "pattern" in output:
logger.opt(colors=True).info("<green>{}</green>".format(output))
elif "rotation" in output:
logger.opt(colors=True).info("<yellow>{}</yellow>".format(output))
elif "Finished" in output:
logger.opt(colors=True).info("<magenta>{}</magenta>".format(output))
elif "Pipeline" in output:
logger.opt(colors=True).info("<red>{}</red>".format(output))
elif "Advisory" in output:
logger.opt(colors=True).info("\033[43m{}\033[0m".format(output))
elif "reslice" in output:
logger.opt(colors=True).info("<blue>{}</blue>".format(output))
else:
logger.info(output)
[docs]
def log_rank(output: Any, comm: MPI.Comm) -> None:
"""
Log output to log file with the process rank.
Parameters
----------
output : Any
The item to be printed.
comm : Comm
The comm used to determine the process rank.
"""
logger.debug(f"RANK: [{comm.rank}], {output}")
[docs]
def log_exception(output: str) -> None:
"""
Log an exception to the log file.
Parameters
----------
output : str
The exception to be logged.
"""
logger.error(output)
def _parse_preview(
preview: List[Dict[str, int]],
data_shape: Tuple[int],
data_indices: List[int],
) -> str:
"""Parse the python list that represents the preview parameter in the loader
into a string that the helper loader functions in
`httomo.data.hdf._utils.load` can understand.
Parameters
----------
preview : List[Dict[str, int]]
A list of dicts, where each dict represents the start, stop, step values
that a dimension of data can be previewed/sliced.
data_shape : Tuple[int]
The shape of the original data to apply previewing to.
data_indices : List[int]
The indices where projection data is in the dataset.
Returns
-------
str
A string that represents the preview parameter from the YAML config.
"""
preview_str = ""
# Pad the `preview` list with None until it is the same length as the number
# of dimensions in the data, since the user may not have specified a preview
# value for every dimension after the last dimension with previewing in the
# YAML config
if len(preview) < len(data_shape):
while len(preview) < len(data_shape):
preview.append(None)
for idx, slice_info in enumerate(preview):
if slice_info is None:
if idx == 0:
# TODO: Assuming data to be stack of projections, so dim 0 would
# be the rotation angle dim, and as such would contain flats and
# darks. These should be cropped out by default if no explicit
# previewing is specified.
#
# However, assumption perhaps should be removed to make this
# function more generic.
preview_str += f"{data_indices[0]}:{data_indices[-1]+1}"
else:
preview_str += ":"
elif slice_info == "mid":
# user can simply write 'mid' to get 3 slices around the middle section of a chosen dimension
mid_slice = data_shape[idx] // 2
if mid_slice > 1:
preview_str += f"{mid_slice-2}:{mid_slice+1}:{1}"
else:
# the size of the dimension is less than 4 so all data can be taken
preview_str += ":"
else:
start = slice_info.get("start", None)
stop = slice_info.get("stop", None)
step = slice_info.get("step", None)
warn_on = False
if start is not None and (start < 0 or start >= data_shape[idx]):
warn_on = True
str_warn = (
f"The 'start' preview {start} is outside the data dimension range"
+ f" from 0 to {data_shape[idx]}"
)
if stop is not None and (stop < 0 or stop >= data_shape[idx]):
warn_on = True
str_warn = (
f"The 'stop' preview {start} is outside the data dimension range"
+ f" from 0 to {data_shape[idx]}"
)
if step is not None and step < 0:
warn_on = True
str_warn = "The 'step' in preview cannot be negative"
if warn_on:
log_exception(str_warn)
raise ValueError("Preview error: " + str_warn)
start_str = f"{start if start is not None else ''}"
stop_str = f"{stop if stop is not None else ''}"
step_str = f"{step if step is not None else ''}"
preview_str += f"{start_str}:{stop_str}:{step_str}"
# Depending on if this is the last dimension in the data or not, a comma
# may or may not be needed
if idx < len(preview) - 1:
preview_str += ", "
return preview_str
def _get_slicing_dim(pattern: Pattern) -> Literal[1, 2]:
"""Assuming 3D projection data with the axes ordered as
- axis 0 = rotation angle
- axis 1 = detector y
- axis 2 = detector x
when given the pattern of a method, return the dimension of the data that
the method requires the data to be sliced in.
"""
if pattern == Pattern.projection:
return 1
elif pattern == Pattern.sinogram:
return 2
elif pattern == Pattern.all:
# this pattern should inherit the pattern from the previous section if it is available.
# It needs taken care of in the runner.
return 1
else:
err_str = f"An unknown pattern has been encountered {pattern}"
log_exception(err_str)
raise ValueError(err_str)
[docs]
def get_data_in_data_out(method_name: str, dict_params_method: Dict[str, Any]) -> tuple:
"""
Get the input and output datasets in a list
"""
if (
"data_in" in dict_params_method.keys()
and "data_out" in dict_params_method.keys()
):
data_in = dict_params_method.pop("data_in")
data_out = dict_params_method.pop("data_out")
else:
# TODO: This error reporting is possibly better handled by
# schema validation of the user config YAML
if method_name != "save_to_images":
if (
"data_in" in dict_params_method.keys()
and "data_out" not in dict_params_method.keys()
):
# Assume "data_out" to be the same as "data_in"
data_in = [dict_params_method.pop("data_in")]
data_out = data_in
else:
err_str = "Invalid in/out dataset parameters"
log_exception(err_str)
raise ValueError(err_str)
else:
data_in = dict_params_method.pop("data_in")
data_out = None
return data_in, data_out
[docs]
class catchtime:
"""A context manager to measure ns-accurate time for the context block.
Usage:
with catchtime() as t:
...
print(t.elapsed)
"""
def __enter__(self):
self.start = perf_counter_ns()
return self
def __exit__(self, type, value, traceback):
self.elapsed = (perf_counter_ns() - self.start) * 1e-9
[docs]
class catch_gputime:
def __enter__(self):
if gpu_enabled:
self.start = xp.cuda.Event()
self.start.record()
return self
def __exit__(self, type, value, traceback):
if gpu_enabled:
self.end = xp.cuda.Event()
self.end.record()
@property
def elapsed(self) -> float:
if gpu_enabled:
self.end.synchronize()
return xp.cuda.get_elapsed_time(self.start, self.end) * 1e-3
else:
return 0.0
[docs]
def make_3d_shape_from_shape(shape: List[int]) -> Tuple[int, int, int]:
"""Given a shape as a list of length 3, return a corresponding tuple
with the right typing type (required to make mypy type checks work)
"""
assert len(shape) == 3, "3D shape expected"
return (shape[0], shape[1], shape[2])
[docs]
def make_3d_shape_from_array(array: np.ndarray) -> Tuple[int, int, int]:
"""Given a 3D array, return a corresponding shape tuple
with the right typing type (required to make mypy type checks work)
"""
return make_3d_shape_from_shape(list(array.shape))
[docs]
def mpi_abort_excepthook(type, value, traceback):
log_rank(f" {type.__name__}: {value}", MPI.COMM_WORLD)
log_rank("\n".join(format_tb(traceback)), MPI.COMM_WORLD)
MPI.COMM_WORLD.Abort()
sys.__excepthook__(type, value, traceback)
[docs]
def search_max_slices_iterative(
available_memory: int, get_mem_bytes: Callable[[int], int]
) -> int:
"""
Approximates the maximum number of fitting slices to the GPU memory for a given function.
The memory profile of the function must be increasing in the function of the number of slices.
First, a linear approximation of the memory profile is performed. If this is not accurate enough,
a binary search follows to determine the number of fitting slices. This function never returns a
number of slices for what `get_mem_bytes(slices) > available_memory`.
:param available_memory: Bytes of available device memory
:type available_memory: int
:param get_mem_bytes: A functor that produces the bytes of device memory needed for a given number of slices.
:type get_mem_bytes: Callable[[int], int]
:return: Returns the approximation of the maximum number of fitting slices.
:rtype: int
"""
MEM_RATIO_THRESHOLD = 0.9 # 90% of the used device memory is the target
# Find a number of slices that does not fit
current_slices = 100
slices_high = None
memory_bytes = get_mem_bytes(current_slices)
if memory_bytes > available_memory:
# Found upper limit, continue to binary search
slices_high = current_slices
else:
# linear approximation
current_slices = int(current_slices * available_memory / memory_bytes)
while True:
memory_bytes = get_mem_bytes(current_slices)
if memory_bytes > available_memory:
# Found upper limit, continue to binary search
break
elif memory_bytes >= available_memory * MEM_RATIO_THRESHOLD:
# This is "good enough", return
return current_slices
# If linear approximation is not enough, just double every iteration
current_slices *= 2
slices_high = current_slices
# Binary search between low and high
slices_low = 0
while slices_high - slices_low > 1:
current_slices = (slices_low + slices_high) // 2
memory_bytes = get_mem_bytes(current_slices)
if memory_bytes > available_memory:
slices_high = current_slices
elif memory_bytes >= available_memory * MEM_RATIO_THRESHOLD:
return current_slices
else:
slices_low = current_slices
return slices_low
[docs]
def make_pinned_host_array(shape, dtype) -> np.ndarray:
if gpu_enabled:
pinned_ptr = xp.cuda.alloc_pinned_memory(
np.prod(shape) * np.dtype(dtype).itemsize
)
return np.frombuffer(pinned_ptr, dtype=dtype, count=np.prod(shape)).reshape(
shape
)
else:
return np.empty(shape, dtype=dtype)
[docs]
def clp2(x: int) -> int:
"""
Round up to next power of two
"""
x -= 1
x |= x >> 1
x |= x >> 2
x |= x >> 4
x |= x >> 8
x |= x >> 16
x |= x >> 32
return x + 1