From fdf31410874616814a0658c86f73d5441d32e570 Mon Sep 17 00:00:00 2001
From: woutdenolf <woutdenolf@users.sf.net>
Date: Fri, 20 May 2022 11:48:37 +0200
Subject: [PATCH] expand task discovery to methods

---
 ewokscore/task_discovery.py            | 67 ++++++++++++++++++++++
 ewokscore/task_summary.py              | 43 --------------
 ewokscore/tests/discover_module.py     | 27 +++++++++
 ewokscore/tests/test_task_discovery.py | 78 ++++++++++++++++++++++++++
 ewokscore/tests/test_task_summary.py   |  8 ---
 5 files changed, 172 insertions(+), 51 deletions(-)
 create mode 100644 ewokscore/task_discovery.py
 delete mode 100644 ewokscore/task_summary.py
 create mode 100644 ewokscore/tests/discover_module.py
 create mode 100644 ewokscore/tests/test_task_discovery.py
 delete mode 100644 ewokscore/tests/test_task_summary.py

diff --git a/ewokscore/task_discovery.py b/ewokscore/task_discovery.py
new file mode 100644
index 0000000..8c3b092
--- /dev/null
+++ b/ewokscore/task_discovery.py
@@ -0,0 +1,67 @@
+import inspect
+import importlib
+import importlib.util
+from typing import Callable, Iterable, Optional
+from .task import Task
+from .utils import qualname
+
+
+def discover_tasks_from_modules(
+    *module_names: Iterable[str], task_type="class"
+) -> Iterable[dict]:
+    if task_type == "method":
+        yield from _iter_method_tasks(*module_names)
+    elif task_type == "ppfmethod":
+        yield from _iter_method_tasks(
+            *module_names, filter_method_name=lambda name: name == "run"
+        )
+    elif task_type == "class":
+        for module_name in module_names:
+            importlib.import_module(module_name)
+        yield from _iter_registered_tasks(*module_names)
+    else:
+        raise ValueError("Class type does not support discovery")
+
+
+def _iter_registered_tasks(*filter_modules: Iterable[str]) -> Iterable[dict]:
+    """Yields all task classes registered in the current process."""
+    for cls in Task.get_subclasses():
+        module = cls.__module__
+        if filter_modules and not any(
+            module.startswith(prefix) for prefix in filter_modules
+        ):
+            continue
+        task_identifier = cls.class_registry_name()
+        category = task_identifier.split(".")[0]
+        yield {
+            "task_type": "class",
+            "task_identifier": task_identifier,
+            "required_input_names": list(cls.required_input_names()),
+            "optional_input_names": list(cls.optional_input_names()),
+            "output_names": list(cls.output_names()),
+            "category": category,
+        }
+
+
+def _iter_method_tasks(
+    *module_names: Iterable[str],
+    filter_method_name: Optional[Callable[[str], bool]] = None
+) -> Iterable[dict]:
+    """Yields all task methods from the provided module_names. The module_names will be will
+    imported for discovery.
+    """
+    for module_name in module_names:
+        mod = importlib.import_module(module_name)
+        for method in inspect.getmembers(mod, inspect.isfunction):
+            method_name, method_qn = method
+            if filter_method_name and not filter_method_name(method_name):
+                continue
+            if method_name.startswith("_"):
+                continue
+            task_identifier = qualname(method_qn)
+            category = task_identifier.split(".")[0]
+            yield {
+                "task_type": "method",
+                "task_identifier": qualname(method_qn),
+                "category": category,
+            }
diff --git a/ewokscore/task_summary.py b/ewokscore/task_summary.py
deleted file mode 100644
index 611e1cc..0000000
--- a/ewokscore/task_summary.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import json
-import logging
-import importlib
-from typing import List, Iterable
-from pathlib import Path
-from .task import Task
-
-logger = logging.getLogger(__name__)
-
-
-def generate_task_summary(*module_names: Iterable[str]) -> List[dict]:
-    filter_packages = set()
-    for module_name in module_names:
-        importlib.import_module(module_name)
-        filter_packages.add(module_name.partition(".")[0])
-    summary = list()
-    for cls in Task.get_subclasses():
-        if filter_packages:
-            package = cls.__module__.partition(".")[0]
-            if package not in filter_packages:
-                continue
-        task_identifier = cls.class_registry_name()
-        category = task_identifier.split(".")[0]
-        info = {
-            "task_type": "class",
-            "task_identifier": task_identifier,
-            "required_input_names": list(cls.required_input_names()),
-            "optional_input_names": list(cls.optional_input_names()),
-            "output_names": list(cls.output_names()),
-            "category": category,
-        }
-        summary.append(info)
-    return summary
-
-
-def save_task_summary(filename, indent=2):
-    summary = generate_task_summary()
-    if not summary:
-        logger.warning(f"No tasks to be saved in {filename}")
-        return
-    filename = Path(filename).with_suffix(".json")
-    with open(filename, "w") as fh:
-        json.dump(summary, fh, indent=indent)
diff --git a/ewokscore/tests/discover_module.py b/ewokscore/tests/discover_module.py
new file mode 100644
index 0000000..6a88b82
--- /dev/null
+++ b/ewokscore/tests/discover_module.py
@@ -0,0 +1,27 @@
+from ewokscore import Task
+
+
+class MyTask1(
+    Task, input_names=["a"], optional_input_names=["b"], output_names=["result"]
+):
+    def run(self):
+        pass
+
+
+class MyTask2(
+    Task, input_names=["a"], optional_input_names=["b"], output_names=["result"]
+):
+    def run(self):
+        pass
+
+
+def run(a, b=None):
+    pass
+
+
+def myfunc(a, b=None):
+    pass
+
+
+def _myfunc(a, b=None):
+    pass
diff --git a/ewokscore/tests/test_task_discovery.py b/ewokscore/tests/test_task_discovery.py
new file mode 100644
index 0000000..1828288
--- /dev/null
+++ b/ewokscore/tests/test_task_discovery.py
@@ -0,0 +1,78 @@
+from ewokscore import task_discovery
+
+
+def test_task_class_discovery():
+    expected = [
+        {
+            "task_type": "class",
+            "task_identifier": "ewokscore.tests.discover_module.MyTask1",
+            "required_input_names": ["a"],
+            "optional_input_names": ["b"],
+            "output_names": ["result"],
+            "category": "ewokscore",
+        },
+        {
+            "task_type": "class",
+            "task_identifier": "ewokscore.tests.discover_module.MyTask2",
+            "required_input_names": ["a"],
+            "optional_input_names": ["b"],
+            "output_names": ["result"],
+            "category": "ewokscore",
+        },
+    ]
+
+    tasks = list(task_discovery.discover_tasks_from_modules())
+    for task in expected:
+        assert task not in tasks
+
+    tasks = list(
+        task_discovery.discover_tasks_from_modules("ewokscore.tests.discover_module")
+    )
+    for task in expected:
+        assert task in tasks
+    assert len(tasks) == len(expected)
+
+    tasks = list(task_discovery.discover_tasks_from_modules())
+    for task in expected:
+        assert task in tasks
+
+
+def test_task_method_discovery():
+    expected = [
+        {
+            "task_type": "method",
+            "task_identifier": "ewokscore.tests.discover_module.run",
+            "category": "ewokscore",
+        },
+        {
+            "task_type": "method",
+            "task_identifier": "ewokscore.tests.discover_module.myfunc",
+            "category": "ewokscore",
+        },
+    ]
+    tasks = list(
+        task_discovery.discover_tasks_from_modules(
+            "ewokscore.tests.discover_module", task_type="method"
+        )
+    )
+    for task in expected:
+        assert task in tasks
+    assert len(tasks) == len(expected)
+
+
+def test_task_ppfmethod_discovery():
+    expected = [
+        {
+            "task_type": "method",
+            "task_identifier": "ewokscore.tests.discover_module.run",
+            "category": "ewokscore",
+        }
+    ]
+    tasks = list(
+        task_discovery.discover_tasks_from_modules(
+            "ewokscore.tests.discover_module", task_type="ppfmethod"
+        )
+    )
+    for task in expected:
+        assert task in tasks
+    assert len(tasks) == len(expected)
diff --git a/ewokscore/tests/test_task_summary.py b/ewokscore/tests/test_task_summary.py
deleted file mode 100644
index fe783df..0000000
--- a/ewokscore/tests/test_task_summary.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from ewokscore.task_summary import generate_task_summary
-from .examples.tasks.sumtask import SumTask
-
-
-def test_graph_summary():
-    summary = generate_task_summary()
-    names = [s["task_identifier"] for s in summary]
-    assert SumTask.class_registry_name() in names
-- 
GitLab