From fdf31410874616814a0658c86f73d5441d32e570 Mon Sep 17 00:00:00 2001 From: woutdenolf <woutdenolf@users.sf.net> Date: Fri, 20 May 2022 11:48:37 +0200 Subject: [PATCH] expand task discovery to methods --- ewokscore/task_discovery.py | 67 ++++++++++++++++++++++ ewokscore/task_summary.py | 43 -------------- ewokscore/tests/discover_module.py | 27 +++++++++ ewokscore/tests/test_task_discovery.py | 78 ++++++++++++++++++++++++++ ewokscore/tests/test_task_summary.py | 8 --- 5 files changed, 172 insertions(+), 51 deletions(-) create mode 100644 ewokscore/task_discovery.py delete mode 100644 ewokscore/task_summary.py create mode 100644 ewokscore/tests/discover_module.py create mode 100644 ewokscore/tests/test_task_discovery.py delete mode 100644 ewokscore/tests/test_task_summary.py diff --git a/ewokscore/task_discovery.py b/ewokscore/task_discovery.py new file mode 100644 index 0000000..8c3b092 --- /dev/null +++ b/ewokscore/task_discovery.py @@ -0,0 +1,67 @@ +import inspect +import importlib +import importlib.util +from typing import Callable, Iterable, Optional +from .task import Task +from .utils import qualname + + +def discover_tasks_from_modules( + *module_names: Iterable[str], task_type="class" +) -> Iterable[dict]: + if task_type == "method": + yield from _iter_method_tasks(*module_names) + elif task_type == "ppfmethod": + yield from _iter_method_tasks( + *module_names, filter_method_name=lambda name: name == "run" + ) + elif task_type == "class": + for module_name in module_names: + importlib.import_module(module_name) + yield from _iter_registered_tasks(*module_names) + else: + raise ValueError("Class type does not support discovery") + + +def _iter_registered_tasks(*filter_modules: Iterable[str]) -> Iterable[dict]: + """Yields all task classes registered in the current process.""" + for cls in Task.get_subclasses(): + module = cls.__module__ + if filter_modules and not any( + module.startswith(prefix) for prefix in filter_modules + ): + continue + task_identifier = cls.class_registry_name() + category = task_identifier.split(".")[0] + yield { + "task_type": "class", + "task_identifier": task_identifier, + "required_input_names": list(cls.required_input_names()), + "optional_input_names": list(cls.optional_input_names()), + "output_names": list(cls.output_names()), + "category": category, + } + + +def _iter_method_tasks( + *module_names: Iterable[str], + filter_method_name: Optional[Callable[[str], bool]] = None +) -> Iterable[dict]: + """Yields all task methods from the provided module_names. The module_names will be will + imported for discovery. + """ + for module_name in module_names: + mod = importlib.import_module(module_name) + for method in inspect.getmembers(mod, inspect.isfunction): + method_name, method_qn = method + if filter_method_name and not filter_method_name(method_name): + continue + if method_name.startswith("_"): + continue + task_identifier = qualname(method_qn) + category = task_identifier.split(".")[0] + yield { + "task_type": "method", + "task_identifier": qualname(method_qn), + "category": category, + } diff --git a/ewokscore/task_summary.py b/ewokscore/task_summary.py deleted file mode 100644 index 611e1cc..0000000 --- a/ewokscore/task_summary.py +++ /dev/null @@ -1,43 +0,0 @@ -import json -import logging -import importlib -from typing import List, Iterable -from pathlib import Path -from .task import Task - -logger = logging.getLogger(__name__) - - -def generate_task_summary(*module_names: Iterable[str]) -> List[dict]: - filter_packages = set() - for module_name in module_names: - importlib.import_module(module_name) - filter_packages.add(module_name.partition(".")[0]) - summary = list() - for cls in Task.get_subclasses(): - if filter_packages: - package = cls.__module__.partition(".")[0] - if package not in filter_packages: - continue - task_identifier = cls.class_registry_name() - category = task_identifier.split(".")[0] - info = { - "task_type": "class", - "task_identifier": task_identifier, - "required_input_names": list(cls.required_input_names()), - "optional_input_names": list(cls.optional_input_names()), - "output_names": list(cls.output_names()), - "category": category, - } - summary.append(info) - return summary - - -def save_task_summary(filename, indent=2): - summary = generate_task_summary() - if not summary: - logger.warning(f"No tasks to be saved in {filename}") - return - filename = Path(filename).with_suffix(".json") - with open(filename, "w") as fh: - json.dump(summary, fh, indent=indent) diff --git a/ewokscore/tests/discover_module.py b/ewokscore/tests/discover_module.py new file mode 100644 index 0000000..6a88b82 --- /dev/null +++ b/ewokscore/tests/discover_module.py @@ -0,0 +1,27 @@ +from ewokscore import Task + + +class MyTask1( + Task, input_names=["a"], optional_input_names=["b"], output_names=["result"] +): + def run(self): + pass + + +class MyTask2( + Task, input_names=["a"], optional_input_names=["b"], output_names=["result"] +): + def run(self): + pass + + +def run(a, b=None): + pass + + +def myfunc(a, b=None): + pass + + +def _myfunc(a, b=None): + pass diff --git a/ewokscore/tests/test_task_discovery.py b/ewokscore/tests/test_task_discovery.py new file mode 100644 index 0000000..1828288 --- /dev/null +++ b/ewokscore/tests/test_task_discovery.py @@ -0,0 +1,78 @@ +from ewokscore import task_discovery + + +def test_task_class_discovery(): + expected = [ + { + "task_type": "class", + "task_identifier": "ewokscore.tests.discover_module.MyTask1", + "required_input_names": ["a"], + "optional_input_names": ["b"], + "output_names": ["result"], + "category": "ewokscore", + }, + { + "task_type": "class", + "task_identifier": "ewokscore.tests.discover_module.MyTask2", + "required_input_names": ["a"], + "optional_input_names": ["b"], + "output_names": ["result"], + "category": "ewokscore", + }, + ] + + tasks = list(task_discovery.discover_tasks_from_modules()) + for task in expected: + assert task not in tasks + + tasks = list( + task_discovery.discover_tasks_from_modules("ewokscore.tests.discover_module") + ) + for task in expected: + assert task in tasks + assert len(tasks) == len(expected) + + tasks = list(task_discovery.discover_tasks_from_modules()) + for task in expected: + assert task in tasks + + +def test_task_method_discovery(): + expected = [ + { + "task_type": "method", + "task_identifier": "ewokscore.tests.discover_module.run", + "category": "ewokscore", + }, + { + "task_type": "method", + "task_identifier": "ewokscore.tests.discover_module.myfunc", + "category": "ewokscore", + }, + ] + tasks = list( + task_discovery.discover_tasks_from_modules( + "ewokscore.tests.discover_module", task_type="method" + ) + ) + for task in expected: + assert task in tasks + assert len(tasks) == len(expected) + + +def test_task_ppfmethod_discovery(): + expected = [ + { + "task_type": "method", + "task_identifier": "ewokscore.tests.discover_module.run", + "category": "ewokscore", + } + ] + tasks = list( + task_discovery.discover_tasks_from_modules( + "ewokscore.tests.discover_module", task_type="ppfmethod" + ) + ) + for task in expected: + assert task in tasks + assert len(tasks) == len(expected) diff --git a/ewokscore/tests/test_task_summary.py b/ewokscore/tests/test_task_summary.py deleted file mode 100644 index fe783df..0000000 --- a/ewokscore/tests/test_task_summary.py +++ /dev/null @@ -1,8 +0,0 @@ -from ewokscore.task_summary import generate_task_summary -from .examples.tasks.sumtask import SumTask - - -def test_graph_summary(): - summary = generate_task_summary() - names = [s["task_identifier"] for s in summary] - assert SumTask.class_registry_name() in names -- GitLab