Source code for httomo_backends.scripts.yaml_templates_generator

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------
# Copyright 2022 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ---------------------------------------------------------------------------
# Created By  : Tomography Team <scientificsoftware@diamond.ac.uk>
# Created Date: 14/October/2022
# version ='0.3'
# ---------------------------------------------------------------------------
"""Script that exposes all functions of a given software package as YAML templates.

Please run the generator as:
    python -m yaml_templates_generator -i /path/to/modules.yml -o /path/to/output/
"""
import argparse
import importlib
import inspect
import os
import re
from typing import Any, List, Dict

import yaml


[docs] def yaml_generator(path_to_modules: str, output_folder: str) -> int: """function that exposes all method of a given software package as YAML templates Args: path_to_modules: path to the list of modules yaml file output_folder: path to output folder with saved templates Returns: returns zero if the processing is succesfull """ discard_keys = _get_discard_keys() no_data_out_modules = _get_discard_data_out() # open YAML file with modules to inspect with open(path_to_modules, "r") as stream: try: modules_list = yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc) # a loop over modules in the file modules_no = len(modules_list) for i in range(modules_no): module_name = modules_list[i] try: imported_module = importlib.import_module(str(module_name)) except NameError: print( "Import of the module {} has failed, check if software installed".format( module_name ) ) methods_list = imported_module.__all__ # get all the methods in the module methods_no = len(methods_list) # a loop over all methods in the module for m in range(methods_no): method_name = methods_list[m] print("Inspecting the signature of the {} method".format(method_name)) get_method_params = inspect.signature( getattr(imported_module, methods_list[m]) ) # get method docstrings get_method_docs = inspect.getdoc(getattr(imported_module, methods_list[m])) # put the parameters in the dictionary params_list: List = [] params_dict: Dict = {} for name, value in get_method_params.parameters.items(): if value is not None: append = True for x in discard_keys: if name == x: append = False break if append: _set_param_value(name, value, params_dict) method_dict = { "method": method_name, "module_path": module_name, "parameters": params_dict, } _set_dict_special_cases(method_dict, method_name) params_list = [method_dict] _save_yaml(module_name, method_name, params_list) return 0
def _set_param_value(name: str, value: inspect.Parameter, params_dict: Dict[str, Any]): """Set param value for method inside dictionary Args: name: Parameter name value: Parameter value params_dict: Dict containing method's parameter names and values """ if value.default is inspect.Parameter.empty and name != "kwargs": if name in ["proj1", "proj2"]: params_dict[name] = "auto" else: params_dict[name] = "REQUIRED" elif name == "kwargs": # params_dict["#additional parameters"] = "AVAILABLE" # parsing hashtag to yaml comes with quotes, for now we simply ignore the field pass elif name == "axis": params_dict[name] = "auto" elif name == "asynchronous": params_dict[name] = True elif name == "center": # Temporary value params_dict[name] = "${{centering.side_outputs.centre_of_rotation}}" elif name == "glob_stats": params_dict[name] = "${{statistics.side_outputs.glob_stats}}" elif name == "overlap": params_dict[name] = "${{centering.side_outputs.overlap}}" else: params_dict[name] = value.default def _save_yaml(module_name: str, method_name: str, params_list: List[str]): """Save the list as a YAML file Args: module_name: Name of module method_name: Name of method params_list: List of parameters """ path_dir = output_folder + "/" + module_name path_file = path_dir + "/" + str(method_name) + ".yaml" if not os.path.exists(path_dir): os.makedirs(path_dir) with open(path_file, "w") as file: outputs = yaml.dump(params_list, file, sort_keys=False) def _set_dict_special_cases(method_dict: Dict, method_name: str): """Dealing with special cases for "data_out" Args: method_dict: Dictionary of modules and parameters method_name: Name of method """ if method_name in ["find_center_vo", "find_center_pc"]: method_dict["id"] = "centering" method_dict["side_outputs"] = {"cor": "centre_of_rotation"} if method_name in "find_center_360": method_dict["id"] = "centering" method_dict["side_outputs"] = { "cor": "centre_of_rotation", "overlap": "overlap", "side": "side", "overlap_position": "overlap_position", } if method_name in "calculate_stats": method_dict["id"] = "statistics" method_dict["side_outputs"] = {"glob_stats": "glob_stats"} def _get_discard_data_out() -> List[str]: """Discard data_out from certain modules Returns: list of data_out to discard """ discard_data_out = ["save_to_images"] return discard_data_out def _get_discard_keys() -> List[str]: """Can work with any software in principle, but for TomoPy, httomolib(gpu) there are additional keys that needed to be discarded in templates in order to let httomo work smoothly. Returns: List of keys to discard """ discard_keys = [ "in_file", "data_in", "tomo", "arr", "prj", "data", "ncore", "nchunk", "flats", "flat", "dark", "darks", "theta", "out", "ang", "comm_rank", "out_dir", "angles", "gpu_id", "comm", "offset", "shift_xy", "step_xy", "jpeg_quality", "watermark_vals", ] return discard_keys
[docs] def get_args(): parser = argparse.ArgumentParser( description="Script that exposes all functions " "of a given software package as YAML templates." ) parser.add_argument( "-i", "--input", type=str, default=None, help="A path to the list of modules yaml file" "which is needed to be inspected and functions extracted.", ) parser.add_argument( "-o", "--output", type=str, default="./", help="Directory to save the yaml templates in.", ) return parser.parse_args()
if __name__ == "__main__": current_dir = os.path.basename(os.path.abspath(os.curdir)) args = get_args() path_to_modules = args.input output_folder = args.output return_val = yaml_generator(path_to_modules, output_folder) if return_val == 0: print("The methods as YAML templates have been successfully generated!")