Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
workflow
pypushflow
Commits
61ac29ae
Commit
61ac29ae
authored
May 13, 2020
by
payno
Browse files
Merge branch 'add_addon' into 'master'
add management of add-ons Closes
#2
See merge request
!3
parents
79bf3958
742a126a
Pipeline
#25823
passed with stages
in 1 minute and 17 seconds
Changes
5
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
pypushflow/AbstractActor.py
View file @
61ac29ae
...
...
@@ -24,8 +24,9 @@ __license__ = "MIT"
__date__
=
"28/05/2019"
import
logging
from
pypushflow.addon
import
utils
from
pypushflow.addon.classes
import
BaseActorAddOn
#from pypushflow import UtilsMongoDb
logger
=
logging
.
getLogger
(
'pypushflow'
)
...
...
@@ -42,6 +43,10 @@ class AbstractActor(object):
self
.
parent
=
parent
self
.
actorId
=
None
self
.
_add_ons
=
[]
for
add_on_class
in
self
.
_getAddOnsClasses
():
self
.
_add_ons
.
append
(
add_on_class
())
def
connect
(
self
,
actor
):
logger
.
debug
(
'Connecting actor "{0}" to actor "{1}"'
.
format
(
self
.
name
,
actor
.
name
...
...
@@ -53,25 +58,30 @@ class AbstractActor(object):
logger
.
debug
(
'In actor "{0}", triggering actor "{1}"'
.
format
(
self
.
name
,
actor
.
name
))
self
.
_process_pre_trigger_add_on
(
inData
)
actor
.
trigger
(
inData
)
# def uploadDataToMongo(self, actorData={}, script=None):
# if self.parent is not None:
# if self.parent.mongoId is not None:
# if self.actorId is None:
# actorPath = self.getActorPath() + '/' + self.name
# self.actorId = UtilsMongoDb.initActor(
# workflowId=self.parent.mongoId,
# name=actorPath,
# actorData=actorData,
# script=script
# )
# else:
# UtilsMongoDb.addDataToActor(
# workflowId=self.parent.mongoId,
# actorId=self.actorId,
# actorData=actorData
# )
self
.
_process_post_trigger_add_on
()
def
getActorPath
(
self
):
return
self
.
parent
.
getActorPath
()
def
get_addons
(
self
):
"""Return the list of add-on that will be apply to this actor"""
raise
NotImplementedError
()
def
_process_post_trigger_add_on
(
self
):
for
add_on
in
self
.
_add_ons
:
add_on
.
post_trigger_action
(
actor
=
self
)
def
_process_pre_trigger_add_on
(
self
,
in_data
):
for
add_on
in
self
.
_add_ons
:
add_on
.
pre_trigger_action
(
actor
=
self
,
in_data
=
in_data
)
def
_getAddOnsClasses
(
self
):
add_ons
=
[]
for
_
,
classes
in
utils
.
get_registered_add_ons_classes
().
items
():
for
class_
in
classes
:
import
inspect
if
BaseActorAddOn
in
(
inspect
.
getmro
(
class_
)):
add_ons
.
append
(
class_
)
return
add_ons
pypushflow/Workflow.py
View file @
61ac29ae
...
...
@@ -31,18 +31,38 @@ from pypushflow.PythonActor import PythonActor as ActorFactory
from
pypushflow.StartActor
import
StartActor
from
pypushflow.StopActor
import
StopActor
from
pypushflow.JoinActor
import
JoinUntilStopSignal
from
pypushflow.addon
import
utils
from
pypushflow.addon.classes
import
BaseWorkflowAddOn
logger
=
logging
.
getLogger
(
'pypushflow'
)
class
Workflow
(
object
):
#from pypushflow import UtilsMongoDb
class
_BaseWorkflow
:
def
__init__
(
self
,
configuration
):
self
.
_configuration
=
configuration
or
{}
self
.
_add_ons
=
[]
for
add_on_class
in
self
.
_getAddOnsClasses
():
self
.
_add_ons
.
append
(
add_on_class
(
workflow
=
self
,
configuration
=
self
.
_configuration
))
def
_getAddOnsClasses
(
self
):
add_ons
=
[]
for
_
,
classes
in
utils
.
get_registered_add_ons_classes
().
items
():
for
class_
in
classes
:
import
inspect
if
BaseWorkflowAddOn
in
(
inspect
.
getmro
(
class_
)):
add_ons
.
append
(
class_
)
return
add_ons
class
Workflow
(
_BaseWorkflow
):
"""TODO"""
def
__init__
(
self
,
name
):
super
(
Workflow
,
self
).
__init__
()
self
.
name
=
name
self
.
listOnErrorActor
=
[]
#self.mongoId = UtilsMongoDb.initMongo(name=name)
def
connectOnError
(
self
,
actor
):
logger
.
debug
(
"In connectOnError in subModule {0}, actor name {1}"
.
format
(
self
.
name
,
actor
.
name
))
...
...
@@ -58,8 +78,16 @@ class Workflow(object):
return
'/'
+
self
.
name
class
ProcessableWorkflow
:
def
__init__
(
self
,
scheme
):
class
ProcessableWorkflow
(
_BaseWorkflow
):
"""Define a workflow that can be executed
:param scheme: the workflow scheme
:param configuration: some configuration / settings that can be pass to
the add-on.
"""
def
__init__
(
self
,
scheme
,
configuration
=
None
):
super
(
ProcessableWorkflow
,
self
).
__init__
(
configuration
=
configuration
)
assert
isinstance
(
scheme
,
Scheme
)
self
.
_representation
=
scheme
# first load node handlers if any
...
...
pypushflow/addon/__init__.py
0 → 100644
View file @
61ac29ae
from
.classes
import
*
\ No newline at end of file
pypushflow/addon/classes.py
0 → 100644
View file @
61ac29ae
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2019-2020 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__
=
[
"H. Payno"
]
__license__
=
"MIT"
__date__
=
"22/04/2020"
class
BaseWorkflowAddOn
:
"""Base class for a Workflow instance"""
def
__init__
(
self
,
workflow_instance
):
pass
class
BaseActorAddOn
:
"""Base class to define an add on for an actor"""
def
__init__
(
self
,
target
):
"""Apply an add on to classes that are instances of target class"""
self
.
__target
=
target
@
property
def
target
(
self
):
return
self
.
__target
def
pre_trigger_action
(
self
,
actor
,
in_data
):
pass
def
post_trigger_action
(
self
,
actor
):
pass
pypushflow/addon/utils.py
0 → 100644
View file @
61ac29ae
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2019-2020 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__
=
[
"H. Payno"
]
__license__
=
"MIT"
__date__
=
"22/04/2020"
import
logging
import
pkgutil
import
inspect
import
importlib
_logger
=
logging
.
getLogger
(
__name__
)
def
get_registered_add_ons
():
"""Return the list of registered add-on"""
try
:
import
ppfaddon
except
ImportError
:
_logger
.
info
(
'no add on found'
)
return
[]
else
:
modules
=
[]
for
importer
,
modname
,
ispkg
in
pkgutil
.
iter_modules
(
ppfaddon
.
__path__
):
if
ispkg
:
modules
.
append
(
modname
)
return
modules
def
_check_module_for_class
(
module
,
depth
=
0
):
classes
=
[]
for
m
in
inspect
.
getmembers
(
module
):
if
inspect
.
isclass
(
m
[
1
]):
if
(
m
[
1
].
__module__
!=
'pypushflow.addon.classes'
):
classes
.
append
(
m
[
1
])
if
inspect
.
ismodule
(
m
[
1
]):
if
inspect
.
isclass
(
m
[
1
]):
classes
.
append
(
m
[
1
])
elif
hasattr
(
m
[
1
],
'__path__'
):
for
importer
,
modname
,
ispkg
in
pkgutil
.
iter_modules
(
m
[
1
].
__path__
):
module_name
=
module
.
__name__
submod_name
=
'.'
.
join
((
module_name
,
m
[
0
],
modname
))
try
:
mod
=
importlib
.
import_module
(
submod_name
)
except
Exception
:
pass
else
:
classes
.
extend
(
_check_module_for_class
(
module
=
mod
,
depth
=
depth
+
1
))
return
classes
def
get_registered_add_ons_classes
():
"""Return a dictionary with the list of registered add-on classes for each
add-on"""
classes
=
{}
add_on_modules
=
get_registered_add_ons
()
for
module_name
in
add_on_modules
:
package_name
=
'.'
.
join
((
'ppfaddon'
,
module_name
))
try
:
module
=
__import__
(
package_name
)
except
:
_logger
.
error
(
'fail to import {}'
.
format
(
package_name
))
return
[]
classes
[
module_name
]
=
_check_module_for_class
(
module
=
module
)
return
classes
def
is_add_on_class_relative_to
(
add_on_class
,
myclass
):
"""Return the list of registered add-on classes"""
return
isinstance
(
myclass
,
add_on_class
.
target
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment