Edna2 logs not visible in /var/log/bes/<workflowName>.log
When looking through /var/log/bes/<workflowName>.log I never see the logs of the Edna2 tasks themselves.
However when I try to emulated a BES /RUN/<workflowName> call myself, I do see print statements and exception tracebacks
from edna2.tasks.AbstractTask import AbstractTask
class MyEdna2Task(AbstractTask):
"""EDNA2 task"""
def run(self, inData):
print(f"THIS IS A PRINT: {inData}")
raise RuntimeError("THIS IS AN ERROR")
def run(**inData):
"""Ewoks task o type 'ppfmethod'"""
diffractionThumbnail = MyEdna2Task(inData=inData)
diffractionThumbnail.execute()
return {"b": 20}
def inject_workflow():
"""Inject a fake BES workflow"""
import json
import pathlib
from bes import flows
module = pathlib.Path(__file__).stem
workflow = {
"graph": {"id": "test_graph"},
"nodes": [
{"id": "1", "task_identifier": f"{module}.run", "task_type": "ppfmethod"}
],
}
flows_dir = pathlib.Path(flows.__file__).parent
with open(flows_dir / "test_graph.json", "w") as f:
json.dump(workflow, f, indent=2)
return "test_graph"
def do_run():
"""BES end-point /RUN/<workflowName> emulation"""
import os
import sys
import json
workflowName = inject_workflow()
requestDict = {"a": 10}
os.makedirs("/var/log/bes", exist_ok=True)
cmd = f"cd {os.getcwd()} && {sys.executable} -m bes.bin.run_ppf_json --inData '{json.dumps(requestDict)}' {workflowName} &"
os.system(cmd)
if __name__ == "__main__":
do_run()(bes) denolf@ideapad3:~/projects/edna2$ python test.py
(bes) denolf@ideapad3:~/projects/edna2$ INFO:pypushflow:['/home/denolf/projects/bes/src/bes/bin/run_ppf_json.py', '--inData', '{"a": 10}', 'test_graph']
DEBUG:pypushflow:********************************************************************************
DEBUG:pypushflow:Before json file search
DEBUG:pypushflow:After json file search
DEBUG:pypushflow:Path graph file: /home/denolf/projects/bes/src/bes/flows/test_graph.json
DEBUG:pypushflow:Time last modified json file: 2024-07-06 22:03:27
DEBUG:pypushflow:Time last modified pickled file: 2024-07-06 21:51:40
DEBUG:pypushflow:Before load_graph
INFO:ewokscore.graph.schema:Graph has no "schema_version": assume version "1.0"
DEBUG:pypushflow:After load_graph
DEBUG:pypushflow:Writing graph to path: /var/log/bes/pypushflow/test_graph.pickle
DEBUG:pypushflow:********************************************************************************
DEBUG:pypushflow:Before execute_graph
INFO:ewokscore.events.global_state:[job '75160b4d-d436-4a81-8a96-bbd0268ffb28'] job started
INFO:pypushflow:[test_graph]
Instantiate new workflow 'test_graph'
DEBUG:pypushflow:[test_graph] add reference to actor 'Start'
DEBUG:pypushflow:[test_graph] add reference to actor 'Stop'
DEBUG:pypushflow:[test_graph] add reference to actor 'Stop on error'
DEBUG:pypushflow:[test_graph] [<ErrorHandler> Stop on error] connect to actor 'Stop'
DEBUG:pypushflow:[test_graph] add reference to actor '1'
DEBUG:pypushflow:[test_graph] [<EwoksPythonActor> 1] connect to error handler 'Stop on error'
DEBUG:pypushflow:[test_graph] [<StartActor> Start] connect to actor '1'
DEBUG:pypushflow:[test_graph] [<EwoksPythonActor> 1] connect to actor 'Stop'
INFO:ewokscore.events.global_state:[job '75160b4d-d436-4a81-8a96-bbd0268ffb28'] [workflow 'test_graph'] workflow started
INFO:pypushflow.concurrent.factory:pypushflow concurrency: scaling
INFO:pypushflow:[test_graph] Execution pool ScalingPool ({'max_workers': None, 'pool_type': None})
DEBUG:pypushflow:[test_graph] Thread started for 'Start' (1 threads running)
INFO:pypushflow:[test_graph] [<StartActor> Start] triggered with inData =
{'_noinput': {'execinfo': {'contexts': {'job': ['75160b4d-d436-4a81-8a96-bbd0268ffb28'],
'workflow': ['test_graph']},
'engine': 'ppf',
'host_name': 'ideapad3',
'job_id': '75160b4d-d436-4a81-8a96-bbd0268ffb28',
'process_id': 62782,
'user_name': 'denolf',
'workflow_id': 'test_graph'},
'varinfo': None},
'a': 10}
INFO:pypushflow:[test_graph] [<StartActor> Start] started
INFO:pypushflow:[test_graph] [<StartActor> Start] finished
DEBUG:pypushflow:[test_graph] Thread started for '1' (2 threads running)
INFO:pypushflow:[test_graph] [<EwoksPythonActor> 1] triggered with inData =
{'_noinput': {'execinfo': {'contexts': {'job': ['75160b4d-d436-4a81-8a96-bbd0268ffb28'],
'workflow': ['test_graph']},
'engine': 'ppf',
'host_name': 'ideapad3',
'job_id': '75160b4d-d436-4a81-8a96-bbd0268ffb28',
'process_id': 62782,
'user_name': 'denolf',
'workflow_id': 'test_graph'},
'node_attrs': {'task_identifier': 'test.run',
'task_type': 'ppfmethod'},
'node_id': '1',
'varinfo': None},
'a': 10}
INFO:pypushflow:[test_graph] [<EwoksPythonActor> 1] started
DEBUG:pypushflow:[test_graph] [<EwoksPythonActor> 1] asynchronous execution of 'ewoksppf.ppfrunscript.run'
INFO:pypushflow.concurrent.factory:pypushflow concurrency: process
INFO:pypushflow.concurrent.process:pypushflow process pool context: 'NoneType'
DEBUG:pypushflow:[test_graph] Thread ended for 'Start' (1 threads running)
DEBUG:pypushflow:[test_graph] [<StopActor> Stop] wait for scheduler threads to be finished
INFO:ewokscore.events.global_state:[job '75160b4d-d436-4a81-8a96-bbd0268ffb28'] [workflow 'test_graph'] [node '1'] [task 'test.run'] task started
THIS IS A PRINT: {'a': 10}
2024-07-06 22:03:27,704 ERROR THIS IS AN ERROR
ERROR:edna2:THIS IS AN ERROR
2024-07-06 22:03:27,704 ERROR Traceback (most recent call last):
File "/home/denolf/projects/edna2/src/edna2/tasks/AbstractTask.py", line 55, in run
billiard.Process.run(self)
File "/home/denolf/virtualenvs/bes/lib/python3.12/site-packages/billiard/process.py", line 110, in run
self._target(*self._args, **self._kwargs)
File "/home/denolf/projects/edna2/src/edna2/tasks/AbstractTask.py", line 111, in executeRun
outData = self.run(inData)
^^^^^^^^^^^^^^^^
File "/home/denolf/projects/edna2/test.py", line 9, in run
raise RuntimeError("THIS IS AN ERROR")
RuntimeError: THIS IS AN ERROR
ERROR:edna2:Traceback (most recent call last):
File "/home/denolf/projects/edna2/src/edna2/tasks/AbstractTask.py", line 55, in run
billiard.Process.run(self)
File "/home/denolf/virtualenvs/bes/lib/python3.12/site-packages/billiard/process.py", line 110, in run
self._target(*self._args, **self._kwargs)
File "/home/denolf/projects/edna2/src/edna2/tasks/AbstractTask.py", line 111, in executeRun
outData = self.run(inData)
^^^^^^^^^^^^^^^^
File "/home/denolf/projects/edna2/test.py", line 9, in run
raise RuntimeError("THIS IS AN ERROR")
RuntimeError: THIS IS AN ERROR
INFO:ewokscore.events.global_state:[job '75160b4d-d436-4a81-8a96-bbd0268ffb28'] [workflow 'test_graph'] [node '1'] [task 'test.run'] task finished
INFO:pypushflow:[test_graph] [<EwoksPythonActor> 1] finished
DEBUG:pypushflow:[test_graph] [<EwoksPythonActor> 1] trigger actor 'Stop' with inData =
{'_noinput': {'execinfo': {'contexts': {'job': ['75160b4d-d436-4a81-8a96-bbd0268ffb28'],
'workflow': ['test_graph']},
'engine': 'ppf',
'host_name': 'ideapad3',
'job_id': '75160b4d-d436-4a81-8a96-bbd0268ffb28',
'process_id': 62782,
'user_name': 'denolf',
'workflow_id': 'test_graph'},
'node_attrs': {'task_identifier': 'test.run',
'task_type': 'ppfmethod'},
'node_id': '1',
'varinfo': None},
'_ppfdict': {'a': 10, 'b': 20}}
DEBUG:pypushflow:[test_graph] Thread started for 'Stop' (2 threads running)
INFO:pypushflow:[test_graph] [<StopActor> Stop] triggered with inData =
{'_noinput': {'execinfo': {'contexts': {'job': ['75160b4d-d436-4a81-8a96-bbd0268ffb28'],
'workflow': ['test_graph']},
'engine': 'ppf',
'host_name': 'ideapad3',
'job_id': '75160b4d-d436-4a81-8a96-bbd0268ffb28',
'process_id': 62782,
'user_name': 'denolf',
'workflow_id': 'test_graph'},
'node_attrs': {'task_identifier': 'test.run',
'task_type': 'ppfmethod'},
'node_id': '1',
'varinfo': None},
'_ppfdict': {'a': 10, 'b': 20}}
DEBUG:pypushflow:[test_graph] Thread ended for 'Stop' (1 threads running)
DEBUG:pypushflow:[test_graph] Thread ended for '1' (0 threads running)
DEBUG:pypushflow:[test_graph] [<StopActor> Stop] scheduler threads are finished
DEBUG:pypushflow:[test_graph] [<StopActor> Stop] finished
INFO:ewokscore.events.global_state:[job '75160b4d-d436-4a81-8a96-bbd0268ffb28'] [workflow 'test_graph'] workflow finished
INFO:ewokscore.events.global_state:[job '75160b4d-d436-4a81-8a96-bbd0268ffb28'] job finished